{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 正规方程"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$\\hat{\\theta} = (X^T\\cdot X)^{-1}\\cdot X^T \\cdot y$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "生成一些近似线性的数据，如下图，来测试一下这个方程。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "X = 2 * np.random.rand(100,1)\n",
    "y = 4 + 3 * X + np.random.randn(100,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_b = np.c_[np.ones((100,1)),X]\n",
    "theta_best = np.linalg.inv(X_b.T.dot(X_b)).dot(X_b.T).dot(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.89233852],\n",
       "       [3.17234501]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta_best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 3.89233852],\n",
       "       [10.23702854]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_new = np.array([[0],[2]])\n",
    "X_new_b = np.c_[np.ones((2,1)),X_new]\n",
    "y_predict = X_new_b.dot(theta_best)\n",
    "y_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deZhU9Z3v8fe3u2lcwigCrogo1+CuSLuUayEILhmTmYxP9I4iMYZJ1InGa0yMyWhmxuV5ZpKYTO48CcmYyI0xi2YyufcmV6GxBKUlggtucUcRFxQlBBcaur/3j1+VVTS9VNc5VXWqzuf1PP1096nl/Op09ef86nt+53fM3RERkebXUu8GiIhIbSjwRURSQoEvIpISCnwRkZRQ4IuIpERbLVc2duxYnzhxYi1XKSLS8FasWPGWu4+L+jw1DfyJEyeyfPnyWq5SRKThmdlLcTyPSjoiIimhwBcRSQkFvohISijwRURSQoEvIpISCnwRkZRQ4IuIpIQCX0QkJRT4IiIpocAXEUmJIQPfzG4xs7Vm9ng/t11pZm5mY6vTPBERiUs5PfyfAKf1XWhmewOnAi/H3CYREamCIQPf3RcDb/dz07eBqwBdFFdEpAFUVMM3s7OANe7+aBn3nWtmy81s+ZtvvlnJ6kREJAbDDnwz2wG4BviHcu7v7vPcvcPdO8aNizyds4iIVKiSHv4kYF/gUTNbBYwHHjKz3eNsmIiIxGvYF0Bx98eAXQu/50O/w93firFdIiISs3KGZd4OdAGTzewVM/tM9ZslIiJxG7KH7+7nDnH7xNhaIyIiVaMzbUVEUkKBLyKSEgp8EZGUUOCLiKSEAl9EJCUU+CIiKaHAFxFJCQW+iEhKKPBFRFJCgS8ikhIKfBGRlFDgi4ikhAJfRCQlFPgiIimhwBcRSQkFvohISijwRURSQoEvIpISCnwRkZRQ4IuIpMSQgW9mt5jZWjN7vGTZv5jZH81spZn9p5ntXN1miohIVOX08H8CnNZn2QLgEHc/DHgGuDrmdomISMyGDHx3Xwy83WfZ3e6+Jf/rA8D4KrRNRERiFEcN/0Lg9wPdaGZzzWy5mS1/8803Y1idiIhUIlLgm9k1wBbgtoHu4+7z3L3D3TvGjRsXZXUiIhJBW6UPNLMLgI8B093d42uSiIhUQ0WBb2anAV8GTnb39+JtkoiIVEM5wzJvB7qAyWb2ipl9BvgeMApYYGaPmNn3q9xOERGJaMgevruf28/i/6hCW0REpIp0pq2ISEoo8EVEUkKBLyKSEgp8EZEq6eqCG28M35Og4nH4IiIysK4umD4duruhvR06OyGTqW+b1MMXEamCXC6EfU9P+J7L1btFCnwRkarIZkPPvrU1fM9m690ilXRERKoikwllnFwuhH29yzmgwBcRqZpMJhlBX6CSjohISijwRURiNthwzP5uq9XwTZV0RERiNNhwzP5ug9oN31QPX0QkRoMNx+zvtlwONm0KyzZtqu7wTfXwRURi0NUVwnrMmNBTL/TYS4djFoZqlt722GPQ2xtu7+0Nj68WBb6ISJkKod53mGXfUs3NN8O6ddver7+hmrkctLSEsG9pCY+rFgW+iEgZBqvN9y3VrFsHV1/d//P0HaqZzcLIkf1/IoibAl9EpAz91d8Lwd1fqaZctTxBS4EvIlKGwUI9amjX6gQtBb6ISB/91eqHCvWqhPaGDbEO21Hgi4iUGKxWX/WeeE8PPPQQ3HUX3H13aMyWLbE9/ZDj8M3sFjNba2aPlyzbxcwWmNmz+e+jY2uRiEgd1XJcPACrV8Mtt8CnPgW77gpHHw1f/zq89x586Utwzz2xraqcHv5PgO8B80uWfQXodPebzOwr+d+/HFurRETqZMyYKo+Lf/ddWLyYrp88Te4eJ/vmL8nwAOy5J5x1FsycCTNmwLhxMa+4jMB398VmNrHP4o8D2fzPtwI5FPgiUmMDjYuPYt26mMfF9/bCypWhRHPXXXDffXR1H8l0OulmJO0jLqXzlpfJ/O1+YBbLaxhIpTX83dz9NQB3f83Mdh3ojmY2F5gLMGHChApXJyKytWpdQnDMmBD0EMbHVzQu/vXXYcGCEPILFsAbb4Tlhx4KX/gCuXVz6Z6/PT09RndvK7nVk8hUN+uBGhy0dfd5wDyAjo4Or/b6RCQdBhsXX6muLrj88tApb20NZ8yW9ZwffAD331882Proo2H5uHFw6qmhTHPqqaFsA2S7oP3ntTnZqlSlgf+Gme2R793vAayNs1EiIkOJcrLTQAo7kd7eUF0ZsJzjDk89VSzT3HsvvP8+jBgBJ5wQ5jqeOROOOKL4caFEva6GVWng/xa4ALgp//2/YmuRSIyqUeOVZKhGaA66E1m3DhYuDCF/993wyith+eTJ8NnPhoA/+WT4yEfKbn+t35PmPniVxcxuJxygHQu8AVwL/Ab4JTABeBk4293fHmplHR0dvnz58ohNFilPtWq8Uh+12nl/uJ7jN5Ohq9iLX7Ei9Ox33jmMopk1K5Rp9tmneo3JM7MV7t4R9XnKGaVz7gA3TY+6cpFqqkaNV+qjGjvvbXYg7vDcc2QeupvMA3fDDYtg48ZQzD/2WPjGN0IvvqMjLGtAOtNWmlY1arxSH3HvvIs7EKe9tYfO079JZuUP4MUXwx323RfOOy/04qdNg512iuNl1J0CX5pWvQ6MSfxi23lv2QLLl5P7+rt0v38yPbTR3ePkfv8+mdMPgyuvDCE/aVKMrU8OBb40tXocGJP4Rdp5v/RSsQ7f2Qnr15MlQ3vLIrrdaG9vIbvga3Bi9eIwKYMHFPgi0hDK3nlv3BjStRDyzzwTlo8fD5/8JMycSWb6dDqf2a5mB4GTMnhAgS91kZQejzSB3l54+OHicMn774fNm2H77cMb7OKLw8HWAw7YauqCWn36S9LgAQW+1FySejyNoFF3jlVt96uvFgN+wQJ4662w/Igj4ItfpGv82eTWH0F2Rlvdt1mSBg8o8KXmktTjSbpG3TnOmweXXBI63yNHxtDu99+HJUuKUxc8np+tfbfd4PTTi1MX7Lbb1tvsxvpvsyQNHlDgS80lqceTdI24c+zqgksvLV63ozCn/LDa7R5CvVCHX7w4PNHIkXDiiTB7dgj5ww7bZobJJG6zpAweUOBLzSWpx5N0le4c61kGyuVC2Ba0tJTZ7rVrw9QFd90VyjSvvRaWH3RQsQ5/0kmwww6DPo06FANT4EtdJKXHk3SV7BzrXQbKZkNHfNOmcELq9743wPo3bYKlS4u9+IcfDsvHjNl6hsnx44e1fnUoBqbAF0m44e4c613SGDBw3eHpp4sHW3O5cPWntjY47ji4/voQ8lOmRJ66oFk6FIVPajBqxzieT4Ev0mSqVdIYTpnow8B9+234VWcx5F9+Odxh//1hzpxwVms2C6NGxdPIJlL6SQ32/2gcz6nAF6mjatTaq1HSKLtMtGULLFtWHE3z4INhqM5OO4Un+OpXQy9+332jN6qOanGMpHAx9XB9Xdt2Uv0KKPClbI06HjyphlNrL2fb971PnH+jQctEL7xQrMMvWgQbNoQjtcccA1//egj4o48OpZsmUKtjJKUXU49Lc/wFpOrqfSCwWZSGcrm19nK2fbX/PluXiZxsyxK45Bch5J9/Ptxpn33gnHNCwJ9yCoweHV8DaqScHWutjpGUXkw9Lgp8KUu9DwQ2g76hfPPNQ9fau7rguuuKH+0H2vZV/fv09JBpXUHneU+QW9BNdvVPyXzlPthxxzB18OWXh5Dff/9txsQ3knJ3mrUa9lkY7RT+rh5L7CvwpSwa27y1SspbfUN53brBa+2FACqEfUvLwNt+/fowCMYspr/P6tXFA60LF8Lbb5MBMlOnwrkzYdY/hwa3t0dcUXKUu9Os1bDP0vV89avPPhPLk7p7zb6mTp3q0riWLnW/4YbwPc2WLnXffnv31tbwvdztMdzH3XBDuC+4t7S4z5zZ/2N+8INwn8LXVVcN/zX5xo3uv/ud+2WXuR94YPHJ9tzTfc4c95/9zH3t2gqeuDEsXer+uc+5jxw5/L9rLQDLPYYMVg9fytboY5vjOuhcafkkkwllnDvvDLP0DvWYvp+qrruu/8fceefWvz/ySBkvorcXVq4sHmy9776wou22CxfivuiiUKY5+OCGLtOUo7SU09oarkc+e3Zjv9cHosCXVIjzoGaU6Q4uvzw8bskSOPTQwdtQbungk58MuV36e79efz1MWVCYumDt2rD80EPhC18IAX/CCWFa4RQp3YEDTJjQnGEPEQPfzL4IXAQ48BjwaXf/II6GicQpzoOapUE8ZkzhTMihn6+SNpTzqWru3PC98Mmh8DsffBB67oVe/MqVYfm4cSHcC1MX7LHH4Ctocmk6PlVx4JvZXsAXgIPc/X0z+yVwDvCTmNomEpu4/6kLITycTw3VDJa5c2HuZx2efBK+nT/Yeu+9YVrhESNCz/2mm0LIH354OAIsQLrm3ola0mkDtjezzcAOwKvRmyQSv2r8Uw+3x16VYHnrrTCKpjCiZs2asPyAA0IxetasUJPfMZapWJpWox+fKlfFge/ua8zsX4GXgfeBu9397r73M7O5wFyACRMmVLo6kcji/qeupMceuQ3d3fDAA8WpC1asCONpRo+GGTOKpRr9r0k/LIz4qeCBZqOBO4FPAeuBXwF3uPtPB3pMR0eHL1++vKL1iSRR1aebcIfnnivW4e+5J1yku7U1rLAQ8B0dkWeYlOQysxXu3hH1eaKUdGYAL7r7m/kG/Ro4Dhgw8EWaTVyfGrbacRy4PsxJUwj5VavCnfbbD84/PwT8tGlhQjKRYYgS+C8Dx5rZDoSSznRA3XeRYepasoXppxrd3Ua7ddPJx8j03h+mDD7lFLjqqhDykybVu6nS4KLU8JeZ2R3AQ8AW4GFgXlwNE2lqL730YR0+938Po3vTV+mhlW5vIzftWjL/uH2YbXLEiHq3VJpIpFE67n4tcG1MbRGJRSKncd64MTSqcLD1mfzUKOPHk51xJO13Q3eP097eRvb6UyEp7ZamojNtpakkZhrn3t5wjdZCwC9dCps3hwtwZ7PFi3IfcAAZMzq7YP78OrRTUkWBLxVJZC+a6kwTXPZrXbOmOHXBwoVhjDyEa7RecUUI+OOPD3Pe9uPWW0Obb71V1xsYrqS+H5NGgS/DVu9e9GD/3HGczVr6/DDIa33/fVi8uDia5oknwvLdd4czzggBP2MG7LbbkOvU9QYqV+/3YyNR4Muw1TOchvrnjno2a9/nv+CC0tfq5H7+Opn7bwshv3hxmKx+5Eg48cRwUe6ZM8NkZMOcYXKgHZV6rkPTzrJ8CnwZtnpONlXOP3eUsfF9n5/33qO9tZ3uXqO9p5vsd/8aeCBMG3zxxWHqghNPDLX5CPrbUannWp40TX4WlQJfhm2gXnQteqNR/rnLaV/2uG7a21rp7oX23s3Mnj+d2UBuhzPIHreZzH+fCzPvgL32ivxa+uq7o1LPtTxpmvwsqoqnVqiEplZoXrXsjVayYxmwfe7w9NPFOnwuR9d7h5FrOYXsYe+QOXt8KNNMmVLzqQvUw5eCJEytIPKhWvZGKynZbN0+J/dvj5O55bsh6F9+Odxp//3hwgvJzJxJJpsNZ7rWkXquEjcFvsRizJgwxbp7feqog/b6N28mO/pJ2u1AumkJtfjb58JOT4Uu9DXXhAuB7LtvbRtdhrRM2yu1ocBvAEkfqVG4dF9PTwj9m2+u/TDNbUofu71QPOlp0SIyGzbQaceRm3Ae2RkjyFz0LTjqKGjTv4Ckh97tCdcIddxCuaS3N4xGXLeuHut3enqM7g96yH3s22Te/hJdHEtu578iO+1YMuf/NzLTp5PZeefaNq6Okt5RkNpT4CdcI4zUqMuwuJ6ecPGPu+4ie8ca2nu+RTcjaPfNZA98g65jfsX0f/9ruv9stN9tdH4ZMsPI+kYPy0boKEjtKfATrtIwrWVg1ezg4urVxdE0CxfCO++AGZmpU+mcPZ/cyJlkz9ubzEn/wo03QvfmynaUSQnLKH/D0o7Cpk1w3XXhS6Gfbgr8CGoRqpWEaT0CqyoHF999N1yIu3C91qeeCsv33BM+8YkwXHL6dBg3jgzFCSa7usLAm8IoyuF+6kjCp6qof8NCR2HTplBqW7gQlixRTz/tmj7wqxXKtQzV4YZpf4FVWJ7oEkVvL6xcWTzYet99dHUfSa5tBtmps8h88yKYNYuuPx1E7l4juw9kxm39FKV/l7a2cB3v2bMrC8t6nrkZdadT6Chcd10I+97e5JYEpXaaOvCrGcpJ6AUOpG9gjRmTjBJFv15/vTjD5IIFsHZtWH7YYXT9zTeZfufn6d7SQvtKo/PbwAaYPmPg11L6d4FwLe/hvtYkjH+PY6eTyYTAX7JE0w5I0NSBX81QTkIvcCB9AytRO6cPPoD77iv24leuDMvHjStekPvUU2GPPcjdCN2/2PaTymCvpbSU0dISdnaVqPf497h2OknYeUlyNHXgVzOUk/6P1Dew6rZzcocnnywebL333hD67e1wwglw000h5A8/PCR0iYH+foO9lkwmnAdwySVhp3D55WHyyqT9fcoR106n3jsvSY6mDvxqh3Kj/CPVfOf01luhcFw42LpmTVh+wAF0nXUjue1PJzt7AplTtq+o3UO9lnXrwn6mHnXrRh/OKc1Nk6dJdN3dIekKAb9iRUjc0aPDBUBmzYJTT6VrzYSaHEuo17DKqOvVzkIGkojJ08xsZ+BHwCGAAxe6e1fURknCucNzzxXr8PfcEy7S3doakuob3wghP3XqVjNM5m6rzbGEepXbohwrScrYf2luUUs63wH+n7v/jZm1A9GuAiHJtX59SKFCL37VqrB80iQ4//xQh582DXbaacCnqOWB7nqU26K8vkQdWJemVXHgm9lfACcBcwDcvRvojqdZUndbtsCDDxYPti5bForio0aFruhVV4WQnzSp7KdM+oHuqKK8viSP+pLmUXEN38yOAOYBTwKHAyuAy9z93T73mwvMBZgwYcLUl156KVKDpYpWrSr24Ds7Q6/eDI4+ujhk8phjYMSIere0KamGLwOJq4YfJfA7gAeA4919mZl9B9jg7l8f6DE6aJswGzeG+nsh5J95JiwfPz7U4GfNCr35XXapbztFUi4JB21fAV5x92X53+8AvhK1QVJFvb3w8MPFg61Ll8LmzeEC3Nls8aLckyeHnn1eNXqe6s2K1F7Fge/ur5vZajOb7O5PA9MJ5Z1YdXXB/Pnh5+HOiSKEMfCFHvyCBcXJ6qdMgSuuCGWa44+HkSP7fXg1Ro9oRIpIfUQdpfP3wG35ETovAJ+O3qSirq7QA+zOHwr+8Y9DBaKZwiH2nu5774XJUwq9+CeeCMt33x3OPDME/IwZsNtuZT1dNUaPaESKSH1ECnx3fwSIXFcaSC4XKg4FzRYOsfR03eGxx4qjaZYsCRPJjBwJJ50Ec+aEkD/00K3KNOWqxrVqNSJFpD4SPbVCNhsGhBR6+PUKh2rVmyvu6a5dG8ozhVLN66+H5QcfHCaRmTkTTjwx1OYjqNa1apt9eKZIUiU68DOZEAr1rOFXs95cdk930ya4//5iwD/8cFg+ZkyYWTI/dQF77RVPw/IKO6RqXKu2UeYhEmkmiQ58qH8wVLPePGBP1x2efrpYh8/lQm2+rS0cYL3++hDyU6ZsM8NknJJUetGoHpHoEh/49Vbt0Ptwh/b22/CrzmLIr14d7vDRj8KFF4YyTTYbznQtU9SQrHfppdD+MWNCaSmpo3q0M5JGocAfQhyh128gbN4cpisoHGx98MHQs99ppzCK5mtfCyE/cWJF7Y6rFFWvT1il7W9pCZ+wkniZPg0xlUaSisCPo6db6T/xVoEwopfOS39D5rn/BYsWwYYNIc2OOQauvTYE/FFHhdJNRI0+9LG0/e5hM5nVv7TUV6NvZ0mXpg/8uvbANmwg9++r6f7gAHq8le6eHnL/+iCZfR6Gc84JdfhTToGdd4591Umqv1eib/tvvjkcNE5a2aTRt7OkS9MHftQe2LA+HfT0hIt/FOrwXV1ke46inU66aad9BGR/ejGcfUNFY+KHo97196gapf2N0k4RSMEVr6L08Mt67OrVxYBfuBDeeSeE+dSpH84w2WXHkbt/hAJBRCqShMnTGkKUHtj8+eF62+4lnw4OezdciLsQ8n/8Y7jznnvCJz5RnLpg7NhiG4DMSTG+KBGRCjR94ENlB127usLcPYVPQK30kL3jcrjuhyH9t98eTj4Z5s4NIX/QQVUv01Rbf+UrDTkUaR6pCPxhe+01ct96jS2bDgdaMXq4sOeHZLYsgcsuCwF/wgmw3Xb1bmls+itfQfQD3tphiCSHAh9C3WbJkuLUBStXkuVY2llEt7XTPsKZfcfZ8Jefr3dLq6a/g9sQ/YC3xqiLJEfTB36/PUx3ePLJYh3+3ntD6Le3h577TTeRmTWLzndHklvckn/s2IFX0gQGGl7Y1hZOeGprG/6QQ41RF0mWpg78rU96cjqvXkjmxZ+FkH/11XCnAw+Ev/u7UKY5+WTYcccPH58BMsfXp+39qWZ5pL+D211dYd8Ixe/DoTHqIsnSvIHf3U3ux6/Q/cE++ZOetpC7dhGZ0f8VZpacOTN8nzCh3i0tSy3KI30PbudyxTNde3qG30PXGHWRZGmewHeHZ58t1uHvuYfsxkOKJz21Qfb758Ocf4bW1kirqseByHqUR+Loodd7tlMRKWrswF+/PnQhCyG/alVYPmkSnH9+qMPv6OQebMuH80GRV1mvA5H1KI+ohy7SXBor8LdsgT/8oRjwy5aFI4qjRoUUvuqqUKqZNOnDh2SAzIz4mlCvA5H1Cl/10EWaR/IDf9Wq4miazk7405/C1IlHHQXXXBMC/phjwrUQa6CeByIVviISReTAN7NWYDmwxt0/FrlFf/5z6MYWQv7ZZ8PyvfeGs88OAT99OuyyS+RVVaJZyxw6QUqk+cXRw78MeAr4i4oe3dsLDz1UvBDI0qWhdLPDDjBtGlx6aQj5yZMTM3VBs/W0dYKUSDpECnwzGw+cCVwPXFH2A9esKdbhFywoXh17yhS48soQ8McdByNHRmmelEknSImkQ9Qe/s3AVcCAF1o1s7nAXICDRo2CQw6BJ54IN+6+O5x5ZrgQyIwZsOuuEZsj/RmqXKMTpETSoeLAN7OPAWvdfYWZZQe6n7vPA+YBdLS0OHvuCXPmhJA/5JDElGmaVTnlmmY9LiEiW4vSwz8eOMvMzgC2A/7CzH7q7ucN+IgjjghlnAbWaAc3yy3XNNtxCRHZVsWB7+5XA1cD5Hv4Vw4a9hCGUw4i6WHaiAc3Va4RkYLEjMNvhDBtxIObKteISEEsge/uOSAX5TkaIUwbtbesco2IQIJ6+I0Qpuoti0gjS0zgN0qYqrcsIo0qMYEPClMRkWoafNiMiIg0DQW+iEhKKPBFRFJCgR+jri648cbwXUQkaRJ10LaRNcKJYyKSburhx6S/E8dERJJEgR+Twoljra3JPXFMRNKtbiWdpE+UNlyNcuKYiKRXXQK/WevdOnFMRJKsLiWdvvXu+fM1ukVEpNrq0sMvnSitrQ1uuSWEfzP19kVEkqYuPfxCvfuf/gk+/ekQ9hrdIiJSXTXt4b/7bijdFA5qZjKhjHPrrcmeFllEpBmYu9dsZS0tHd7Ssnyb0k2zjdgREYmTma1w946oz1PTHr57/1e00ugWEZHqq2ngm4XrmKt0IyJSezUN/MmTYfZslW5EROqh4sA3s72B+cDuQC8wz92/M9hjdtwRrr660jWKiEgUUXr4W4D/4e4PmdkoYIWZLXD3J2Nqm4iIxKjicfju/pq7P5T/+c/AU8BecTVMRETiFcuJV2Y2EZgCLOvntrlmttzMlr/55ptxrE5ERCoQOfDN7CPAncDl7r6h7+3uPs/dO9y9Y9y4cVFXJyIiFYoU+GY2ghD2t7n7r+NpkoiIVEPFgW9mBvwH8JS7fyu+JomISDVE6eEfD5wPnGJmj+S/zoipXSIiErOKh2W6+32AxdgWERGpIl3TVkQkJRT4IiIpocAXEUkJBb6ISEoo8EVEUkKBLyKSEgp8EZGUUOCLiKSEAl9EJCUU+CIiKaHAFxFJCQW+iEhKKPBFRFJCgS8ikhIKfBGRlFDgi4ikhAJfRCQlFPgiIimhwBcRSQkFvohISkQKfDM7zcyeNrPnzOwrcTVKRETiV3Hgm1kr8D+B04GDgHPN7KC4GiYiIvGK0sM/GnjO3V9w927g58DH42mWiIjErS3CY/cCVpf8/gpwTN87mdlcYG7+101m9niEddbKWOCtejeiDGpnfBqhjaB2xq1R2jk5jieJEvjWzzLfZoH7PGAegJktd/eOCOusCbUzXo3QzkZoI6idcWukdsbxPFFKOq8Ae5f8Ph54NVpzRESkWqIE/oPA/ma2r5m1A+cAv42nWSIiEreKSzruvsXMLgXuAlqBW9z9iSEeNq/S9dWY2hmvRmhnI7QR1M64paqd5r5N2V1ERJqQzrQVEUkJBb6ISErEFvhDTbNgZiPN7Bf525eZ2cSS267OL3/azGbF1aYK2niFmT1pZivNrNPM9im5rcfMHsl/VfXgdBntnGNmb5a056KS2y4ws2fzXxfUuZ3fLmnjM2a2vuS2mmxPM7vFzNYOdP6HBd/Nv4aVZnZkyW213JZDtfNv8+1baWZLzezwkttWmdlj+W0Zy/C9CO3MmtmfSv62/1ByW82mYimjnV8qaePj+ffjLvnbarI9zWxvM7vHzJ4ysyfM7LJ+7hPv+9PdI38RDto+D+wHtAOPAgf1uc/FwPfzP58D/CL/80H5+48E9s0/T2sc7aqgjdOAHfI/f77QxvzvG+NuU4R2zgG+189jdwFeyH8fnf95dL3a2ef+f084sF/r7XkScCTw+AC3nwH8nnBeybHAslpvyzLbeVxh/YTpTJaV3LYKGJuQ7ZkF/k/U90u129nnvn8JLKr19gT2AI7M/zwKeKaf//VY359x9fDLmWbh48Ct+Z/vAKabmeWX/9zdN7n7i8Bz+eeL25BtdPd73P29/K8PEM4tqLUoU1bMAha4+9vu/g6wADgtIe08F7i9Sm0ZkLsvBt4e5C4fB+Z78ACws9r+0FsAAANoSURBVJntQW235ZDtdPel+XZA/d6b5WzPgdR0KpZhtrNe783X3P2h/M9/Bp4izGBQKtb3Z1yB3980C30b/uF93H0L8CdgTJmPrVUbS32GsGct2M7MlpvZA2b2iSq0r6Dcdn4y/xHvDjMrnABXq205rHXlS2P7AotKFtdqew5loNdRy205XH3fmw7cbWYrLExlUm8ZM3vUzH5vZgfnlyVye5rZDoSgvLNkcc23p4US9xRgWZ+bYn1/RplaoVQ50ywMdJ+ypmiIQdnrMbPzgA7g5JLFE9z9VTPbD1hkZo+5+/N1auf/Bm53901m9jnCJ6dTynxsXIazrnOAO9y9p2RZrbbnUOr9vhwWM5tGCPwTShYfn9+WuwILzOyP+R5uPTwE7OPuG83sDOA3wP4kdHsSyjn3u3vpp4Gabk8z+whhh3O5u2/oe3M/D6n4/RlXD7+caRY+vI+ZtQE7ET5y1WqKhrLWY2YzgGuAs9x9U2G5u7+a//4CkCPsjathyHa6+7qStv0QmFruY2vZzhLn0Ocjcw2351AGeh2JmzrEzA4DfgR83N3XFZaXbMu1wH9SnZJoWdx9g7tvzP/8O2CEmY0lgdszb7D3ZtW3p5mNIIT9be7+637uEu/7M6aDD22Egwb7Ujwgc3Cf+1zC1gdtf5n/+WC2Pmj7AtU5aFtOG6cQDizt32f5aGBk/uexwLNU6YBTme3co+TnvwIe8OKBnBfz7R2d/3mXerUzf7/JhINgVo/tmV/HRAY+yHgmWx8U+0Ott2WZ7ZxAOL51XJ/lOwKjSn5eCpxWx3buXvhbE4Ly5fy2Lev9Uqt25m8vdDp3rMf2zG+X+cDNg9wn1vdnnI0/g3CU+XngmvyyfyT0lAG2A36Vf9P+Adiv5LHX5B/3NHB6Fd8AQ7VxIfAG8Ej+67f55ccBj+XfpI8Bn6nyG3Wodt4IPJFvzz3AASWPvTC/jZ8DPl3PduZ/vw64qc/jarY9Cb2314DNhF7RZ4DPAZ/L326EC/k8n29LR5225VDt/BHwTsl7c3l++X757fho/j1xTZ3beWnJe/MBSnZQ/b1f6tXO/H3mEAaMlD6uZtuTUJZzYGXJ3/WMar4/NbWCiEhK6ExbEZGUUOCLiKSEAl9EJCUU+CIiKaHAFxFJCQW+iEhKKPBFRFLi/wOCdSC9p9cjaQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(X_new, y_predict, \"r-\")\n",
    "plt.plot(X, y, \"b.\")\n",
    "plt.axis([0,2,0,15])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([3.89233852]), array([[3.17234501]]))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "lr = LinearRegression()\n",
    "lr.fit(X,y)\n",
    "lr.intercept_,lr.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 3.89233852],\n",
       "       [10.23702854]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.predict(X_new)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 梯度下降"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$\\nabla_{\\theta} = \\begin{pmatrix}\\frac{\\partial}{\\partial \\theta_0}MSE(\\theta)\\\\\n",
    "    \\frac{\\partial}{\\partial \\theta_1}MSE(\\theta)\\\\\n",
    "    \\vdots\\\\\n",
    "    \\frac{\\partial}{\\partial \\theta_n}MSE(\\theta)\\\\ \\end{pmatrix}= \\frac{2}{m}X^T\\cdot(X \\cdot \\theta - y)$ "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$\\theta^{t} = \\theta^{t-1} - \\eta\\nabla_{\\theta}MSE(\\theta^{t-1})$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "eta = 0.1 # 学习率\n",
    "n_iterations = 1000 # 迭代次数\n",
    "m = 100\n",
    "theta = np.random.randn(2,1) # 随机初始值\n",
    "\n",
    "for iteration in range(n_iterations):\n",
    "    gradients = 2/m * X_b.T.dot(X_b.dot(theta) - y)\n",
    "    theta = theta - eta * gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.89233852],\n",
       "       [3.17234501]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 随机梯度下降"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{definition}[模拟退火]\n",
    "虽然随机性可以很好的跳过局部最优值，但同时它却不能达到最小值。解决这个难题的一个办法是逐渐降低学习率。 开始时，走的每一步较大（这有助于快速前进同时跳过局部最小值），然后变得越来越小，从而使算法到达全局最小值。 这个过程被称为模拟退火\n",
    "\\end{definition}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 50\n",
    "t_0,t_1 = 5,50 # learning_schedule 的超参数\n",
    "\n",
    "def learning_schedule(t):\n",
    "    return t_0/(t + t_1)\n",
    "\n",
    "theta = np.random.randn(2,1)\n",
    "\n",
    "for epoch in range(n_epochs):\n",
    "    for i in range(m):\n",
    "        random_index = np.random.randint(m)\n",
    "        x_i = X_b[random_index:random_index + 1]\n",
    "        y_i = y[random_index:random_index + 1]\n",
    "        gradients = 2 *x_i.T.dot(x_i.dot(theta) - y_i)\n",
    "        eta = learning_schedule(epoch * m +i)\n",
    "        theta = theta - eta * gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3.90586702],\n",
       "       [3.17656054]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "通过使用 Scikit-Learn 完成线性回归的随机梯度下降，你需要使用SGDRegressor类，这个类默认优化的是均方差损失函数。下面的代码迭代了 50 代，其学习率 $\\eta$ 为 0.1（eta0=0.1），使用默认的learning schedule（与前面的不一样），同时也没有添加任何正则项（penalty = None）："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SGDRegressor(alpha=0.0001, average=False, early_stopping=False, epsilon=0.1,\n",
       "             eta0=0.1, fit_intercept=True, l1_ratio=0.15,\n",
       "             learning_rate='invscaling', loss='squared_loss', max_iter=50,\n",
       "             n_iter_no_change=5, penalty=None, power_t=0.25, random_state=None,\n",
       "             shuffle=True, tol=0.001, validation_fraction=0.1, verbose=0,\n",
       "             warm_start=False)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import SGDRegressor\n",
    "sgd_reg = SGDRegressor(max_iter=50, penalty=None, eta0=0.1)\n",
    "sgd_reg.fit(X,y.flatten())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([3.86991609]), array([3.17455751]))"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_reg.intercept_, sgd_reg.coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 小批量梯度下降"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "训练期间三种梯度下降算法在参数空间中所采用的路径。 他们都接近最小值，但批量梯度的路径最后停在了最小值，而随机梯度和小批量梯度最后都在最小值附近摆动。 但是，不要忘记，批量梯度需要花费大量时间来完成每一步，但是，如果你使用了一个较好的learning schedule，随机梯度和小批量梯度也可以得到最小值。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 多项式回归"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先，我们根据一个简单的二次方程（并加上一些噪声，如图 4-12）来生成一些非线性数据："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAYY0lEQVR4nO3df4wc9XnH8c/j44AzSXqk+I/4sDGVItNQJ3ZzSmhdRQn5YRQIXIkqEpGoav+w8kcaGhE3TpPGtE2EK6qQqOofRSX9IVESElMLCVqSyqRtkKCcsSkBQ0WDAj5oc2lyJJRrcj4//cO39t7ezO7MznxnvjP7fkmWfHd7u9/ZvX32O8/3medr7i4AQLzW1T0AAEB/BGoAiByBGgAiR6AGgMgRqAEgcmeFuNMLLrjAt2zZEuKuAaCVDh8+/AN335D0syCBesuWLZqdnQ1x1wDQSmb2vbSfkfoAgMgRqAEgcgRqAIgcgRoAIkegBoDIEagBIHJByvMAoI0OHpnTLfc/rRcWFrVxckJ7dm3VzI6p4I9LoAaADA4emdOn7n5ci0vLkqS5hUV96u7HJSl4sCb1AQAZ3HL/06eDdMfi0rJuuf/p4I9NoAaADF5YWMz1/TKR+gCADDZOTmguIShvnJwInrtmRg0AGezZtVUT42OrvmeStvz8hD519+OaW1iU60zu+uCRudIem0ANABnM7JjS+988Jev6nkt68D9/GDx3TaAGgIweeGpeWbcDLzN3TaAGgIzyBN+NkxOlPS6BGgAyyhp8J8bHtGfX1tIel0ANABklLSj2mpqc0M3Xbiu16oPyPADIqBN8b7rnCS0sLq362cT4WOkBuoMZNQDkMLNjSkf3vUdfvG67piYnZAozi+7GjBoAhjCzY6qShkwSgRoAMqmrc55EoAaAgersnCeRowaAgersnCcxowYwovKkMursnCcxowYwgjqpjKyNlNIudCnz6sN+CNQARk7eVEbShS5lX33YD6kPACMnbyqjkxKh6gMAKpK2CcDk+vHU36mybroXqQ8AI2fPrq0aH7M133/5/06U2vC/LARqACNnZseUzjt7bUJh6aRXVnKXB4EawEh6qaepUkdVJXd5EKgBjKS00rp1ZtGlPwjUAEZSWm/pZffSN6ctikANYCTN7JjSzddu05itXVSs8vLwLAjUAEbWzI4pnfTk7WpjylVnCtRm9nEze8LMvmNmd5rZuaEHBgBVSKud7ldTXbWBF7yY2ZSkj0l6g7svmtldkj4g6a8Djw0AgkuZUCd+v66e1FmvTDxL0oSZLUlaL+mFcEMCgOqklen1fr/OntQDUx/uPifpTyU9J+lFSS+5+zd6b2dmu81s1sxm5+fnyx8pAASQtTNenT2pBwZqMztf0jWSLpa0UdJ5Zvah3tu5+23uPu3u0xs2bCh/pACQ0cEjc9q5/5Au3nuvdu4/1LfULmtnvDp7UmdZTHyXpGfdfd7dlyTdLelXww4LAIaTt9d0p0xv0I7idfakzpKjfk7SZWa2XtKipHdKmg06KgAYUr8URVouOUtnvD27tq7KUUvV9aQeGKjd/WEz+7qkRyWdkHRE0m2hBwYAwwiVoqizJ3Wmqg933ydpX+CxAEBhab2my0hR1NWTmisTAbRK3dtmhcAOLwBao3NByuLSssbMtOyuqYq3zQqBQA2gFXovSFl2Pz2TbnKQlkh9AGiJOi9ICY1ADaAV6rwgJTRSHwBaYZidxbOqqxlTBzNqAK0QamfxvFc6hkCgBtAKeXcWz9oPJIbcN6kPAK2R1rJ0bmFRO/cfOp26eMclG3Tg8FymlqUx5L6ZUQNojbSrD01albq446HnMs+S62zG1EGgBtAaSVclmqTezVpSNnVJnCXHcKUjqQ8ArZHUOCmpEiRN0iy5zmZMHeZpG4YVMD097bOzdEIFUL+d+w8lBuvemfbE+FhiH+qqmNlhd59O+hmpDwCtlpa6uP6yzQM3C4gFqQ8ArRZD6qIoAjWA1qurj3RZSH0AQOQI1AAQOQI1AESOQA0AkSNQA0DkCNQAEDnK8wBEr+7G/XUjUAOIWu+mtf1akrYVqQ8AUYuhcX/dmFEDiFrRxv1tSJswowYQtSKN+2PY77AMBGoAUSvSuL8taRNSHwCiTg8U6X6XtmlAlfsdloFADYy4JlRVDNP97uCRucRtuKRq9zssA6kPYMS1JT3Q65b7n04M0iZVut9hGTIFajObNLOvm9lTZnbMzH4l9MAAVKNoVUWs0sbviudMIausM+ovSfpHd79E0pskHQs3JABVKlJVEbO08U818LgGBmoze42kt0m6XZLc/WfuvhB6YACqUaSqImZtOq4si4m/IGle0l+Z2ZskHZZ0g7v/b/eNzGy3pN2StHnz5rLHCSCQNuwpmKRNx2XuSen2rhuYTUt6SNJOd3/YzL4k6cfu/gdpvzM9Pe2zs7PljhQAWszMDrv7dNLPsuSoj0s67u4Pr3z9dUm/XNbgAAD9DQzU7v5fkp43s05i552Sngw6KgDAaVkvePkdSXeY2dmSvivpt8INCQDQLVOgdvejkhJzJwCAsLgyEQAiR68PAKWJublTkxGoAZQiVHMngj+BGkBJ+jV36g6seQJvEzr7VYEcNYBSZGnulHfHlbZ29suLQA2gFFmaO+UNvG3t7JcXgRpAKbI0QcobeNva2S8vAjWAVAePzGnn/kO6eO+92rn/UN9NYWd2TOnma7dpanJCplPtRG++dtuqXHLewNumDnhFDGzKNAyaMgHN17uQJ53aHeX6yzbrczPbSrvPifGxNQG993dGoeqjX1Mmqj4AJErKJ7ukOx56TtMXvXaoYDlM69Fh9ktsGwI1gET9trLqLbnLg8CbHzlqAIn6LdiNWtVF3QjUABLt2bVVlvKzUau6qBuBGkCimR1Tuv6yzWuC9ShWXdSNQA0g1edmtunW67b3LblDeCwmAuiLxb/6EagB9DUqdcwxI1ADSEX3ujiQowaQiu51cSBQA0hF97o4ND71Qf4MCGfj5ITmEoIyddTVavSMOm8TcgD50L0uDo2eUWfd+gfAcPo1UeJstjqNDtQh8mf88QGrJdVRf+bg47rjoefUaZJMNUhYjQ7UZefPKEVCr7Z8cJd5HAePzK0K0h2czYbT6Bx12fkzSpHQrS1rIGUfxy33P70mSHdQDRJGowN1lq1/8qAUCd3a8sFd9nH0ez9QDRJGo1MfUrl9CChFQre6PrjLTreUfRxp7xOTqAYJpNEz6rJRioRuVe6A3dlEdsvee/Xxrx4tNd1S9nEkvU86eymSnw4j6kCdZwfkMpSdSkGzVfXB3Z1DlpS6SDesYY8j7f2X9D659brtQ294i8Gi2YW893TvHZds0IHDc7l2KwbKVkXVx879hxJTCd1M0rP7r1w1prmFRY2ZadldUwlj6x775PpxuUsvLS5lOo5hdgtHMaXsQm5mY5JmJc25+1VlDU5KLouj/AcxqKIXc5ZccSdN0fteWV6ZaPWWkvbe7kevLGlifEy3Xrc90/FwMVlc8qQ+bpB0LMQg0ralT0IFBtpmUK64O02R9F7p6E6RFK30oAIqLpkCtZldKOlKSX8ZYhB5XnwqMNA2aYtz0tp1kkHvlc7PiwbaKhdSMVjW1McXJf2epFeHGES/cp/umTUVGGijfv00eqW9V7p/3u92WQPtnl1bE3PUvP/qMXBGbWZXSfq+ux8ecLvdZjZrZrPz8/O5BpG2Kn39ZZupwMBImNkxpQf3Xq5n91+pB/denvp3nvRe6egOpEUrVqiAisvAqg8zu1nShyWdkHSupNdIutvdP5T2O2VUfTS1pwKS8fqWZ5iqD57z+PWr+shVnmdmb5f0iUFVH8MEarQXpV6rEUCRpF+gjvqCF7RDW3pmlKEtjZ5QrVyB2t2/VXYNNdqPUq8z+NDCMJhRIzhKvc7gQwvDIFAjOJpdncGHFoZBoEZwlHqdMcyHVtXNyRCfxvejbqu2VQZU0TOjCfJuFiuJ7eEQT/c8nEE52+hJe83PHV+nH72ytOb2U5MTenDv5VUOEYFRntcwVAaMnrTXPClISyw+jhoCdYSoDBg9eV9bFh9HCznqLqHywnnvl70bR0/aaz45Ma6fnjhJc6QRx4x6Ragrxoa5X8rZRk/aa37T1ZdSMQNm1B2hdrQY5n7ztL1EOwx6zXntRxuBekWovPCw90s52+jhNUcaAvWKUHlh8s3t1W/toW118KgXOeoVofLC5Jvbqd/aAx3yUDZm1CtC5YXJN7fToFp3dvBGmQjUXULlCMk9ts8waw/UwWNYBGq0ThX54UFrD6xLoEzkqFG7MrvDVZUf7rf2wLoEysaMGrXqbUZUtDtcqHr4XlnWHupYl6DapJ3onoda7dx/KDFNMGx3uIv33qukv2iT9Oz+K/MPsKAqAyddF5utX/e8xs2omTG0S9kXGpVRt17W31jZZwuDVHU2geo1Kkc9SvWpo7KrR9lbUxXND5f5N1Z1u1q6LrZXowL1qPRpHqUPpLIX3opu+1Xm31jVgZP9GNurUamPUZkxFD2FbVJ6KMQFQUXq1sv8G6u6fcCeXVsTc9RUmzRfowL1qPTNKBIsqs6LliGmC4LK/BurOnByFWx7NSpQD/uHX/XKe97H6v2dn5sY18Li2i2YsgQLFpSKKTO41hE4Y/rQQ3kaFaiH+cOvcoY5zGMl/c74mGl8nWnp5JlCs6zBInR6qElplWGUHVwJnChDowK1lP8Pv8oZ5jCPlfQ7S8uu89ePa/3ZZ+UOFiHTQ6E/9GL5ECC4IjaNC9R5VbkAWWajnoVXlnTks+/JPYaQedEsH0TDBtsm5taBqjSqPG8YVZYsDfNYZY+vaHlaP4M+iIqUFY5K6SUwjNbPqKtceR/0WEmzzRDjC3XqPiitUiTNlPdsJJY0CVCF1gfqKlfe+z1W2qn9zddu083Xbosm6PQGwHdcskEPPDWvFxYWNbl+vO8iZ5E0U57ceh1pEj4YUCeaMlWk7OZDISQ19ek1PmY67+yz9NLi0pqAVeQY8zQUqvq5pNkRqtCvKdPAHLWZbTKzB8zsmJk9YWY3lD/E9mvCVZVJqYteS8uu8845S8/uv1IP7r18VaAqcjl4ntx61c8l+XPULUvq44SkG939UTN7taTDZvZNd38y8NhapQlXVWYNdGm3K5pmyppbr/q5bMKHLNptYKB29xclvbjy/5+Y2TFJU5II1Dk0oQ9DWgBMul2aKmqQq34um/Ahi3bLVZ5nZlsk7ZD0cMLPdpvZrJnNzs/PlzO6FglZNleWpNRFrxg+XKp+LpOeF9OpRcw2t6BFPDIvJprZqyT9s6TPu/vd/W7LYmJzHTwyp5vueeJ0r5H14+t0zviYFl5Zu3iY9vttrI7oHNfcwqJMWrWLTO/CYlufA4RVeIcXMxuXdEDSHYOCdJV4Q4Tx0xMnT///laWTcpluvW57puZSZZTNxfi6dlI6SRUn3bXiXGGJELJUfZik2yUdc/cvhB9SNmU21x+V3VSyKFLhUEZ1ROybJgxaWKRCBCFkyVHvlPRhSZeb2dGVf+8NPK6BynpDxB4YqlakwqGM6ojYA92gS/6pEEEIAwO1u3/b3c3d3+ju21f+3VfF4Pop6w0Re2BIE+osoEjvkTL6lsQe6AbVirMdFkJobFOmvG+ItMAWe2BIEvIsoMhFK2Xsfxh7oBtUcVL2HpCA1OBeH3lqafst8DSxRjZkj+0iF62U0VelCfXm/WrF2Q4LITS610fW6oB+vSHSAsMwdblVVStcvPdepb1qJjU+OHzm4OO68+HnteyuMTN98K2b9LmZbXUPCwiqcHlerLJeBdcvvVHWDKjKsqx+VxB2p0JCPHZoB4/M6cDhOS2vTCCW3XXg8JymL3pt444FKEujA3VWg9IbZVz2XOWWX0lnAb2auqFtXZvzxli7DXQ0djExjyoWeKpclOxd0Mo7ppjVsbhLiSZiNxIz6ioWeKpelOw+C0jLwce0IJp1xjrs81hkRlzXLB7IaiQCtRS+q1ud1QqxV0rkyd8PcyxF1weaWKKJ0TISqY8q9KYjzl8/rnPOWqePf/Vo8MvSY+/Ml+eiomGOJe3+b7zrsUzPe+y128DIzKir0Jm119GYp4o+0MPKO2PNeyxp97Psnul5j/2MBGBGHUAdl6XH3Fgq9Iy13/1ked5jPyMBmFEHUHXOM/bWmqFnrIPKFbM87zGfkQDMqAOoOucZe2Op0DPWzv2PWXKxIrlmNB0z6gCqznk2oWoh9Iy1c9/kmtFGBOoAqm7Mk6f2uM1X4NEQCW3V6KZMOKU3Ry0lN5bKejsA1evXlIkcdQtkzQHHnssGkIzUR0tkyQE3IZcNYC1m1CMkrfphnVlUddcAVmNG3UBpC4KDFgrT6o2zXsEXUpsXOYGiWExsmLQFwfe/eUoHDs9lWlC88a7HTjfm7zY1OaEH916eeRxlBdaDR+a052uPaenk6jFNTozrpqsvJWBjJLCY2CJpC4J3Pvx8poXCmR1TOpny4Zw1V112/+ab7nliTZCWpIXFJfpCAyJQN06/BkRZb1/0ysmyq0cWFpdSf0ZVCkCgbpy0YJrn8umiO95UXT1CVQpGHYG6YdKC7Affuilz8C3ae6PsXibnrx8f6vGAUUHVR02GXYzrd5n09EWvzXyfRXpvlN3LZN/7LtWNX3tMywl5anp1AFR91KINl3JXVfVx1ZtepweemqdsD63Xr+qDQD2kIoEqbTPaPOVxbZL2fExOjOunJ042+gMNyIryvJIVLU/jUu7V0o57YXGJ3iSACNRDKVqexmaqq+U97lH9QMPoIlAPoeiMuGh5XNukPR9p1SCj+oGG0ZUpUJvZFWb2tJk9Y2Z7Qw8qdkVnxGymulra87HvfZfygQYoQ3memY1J+nNJ75Z0XNIjZnaPuz8ZenCxKqM8jc1UV+v3fNCsCaMuSx31WyQ94+7flSQz+4qkaySNbKBmy6fq8IEGZAvUU5Ke7/r6uKS39t7IzHZL2i1JmzdvLmVwMSOAAKhKlkCd1ERiTfG1u98m6TbpVB11wXEBlaEXNmKXJVAfl7Sp6+sLJb0QZjhAtXqvEu3UxEv1baIA9MpS9fGIpNeb2cVmdrakD0i6J+ywgGqw4S+aYOCM2t1PmNlHJd0vaUzSl939ieAjwxqcopePq0TRBJm657n7fZLuCzwW9MEpehgbJycS+4xwUQ1iwpWJDcEpehhcJYomoB91Q3CKHgY18WgCAnVDcIoeDjXxiB2pj4bgFB0YXcyoG4JTdGB0EagbhFN0YDSR+gCAyBGoASByBGoAiByBGgAiR6AGgMiZe/mto81sXtL3Un58gaQflP6g9eKY4te245E4pibIczwXufuGpB8ECdT9mNmsu09X+qCBcUzxa9vxSBxTE5R1PKQ+ACByBGoAiFwdgfq2Gh4zNI4pfm07HoljaoJSjqfyHDUAIB9SHwAQOQI1AESulkBtZn9sZv9uZkfN7BtmtrGOcZTJzG4xs6dWjuvvzWyy7jEVZWa/YWZPmNlJM2tsyZSZXWFmT5vZM2a2t+7xFGVmXzaz75vZd+oeSxnMbJOZPWBmx1b+3m6oe0xFmdm5ZvZvZvbYyjH9YaH7qyNHbWavcfcfr/z/Y5Le4O4fqXwgJTKz90g6tLJr+59Ikrt/suZhFWJmvyjppKS/kPQJd5+teUi5mdmYpP+Q9G5JxyU9IumD7v5krQMrwMzeJullSX/r7r9U93iKMrPXSXqduz9qZq+WdFjSTMNfI5N0nru/bGbjkr4t6QZ3f2iY+6tlRt0J0ivOk9T4FU13/4a7n1j58iFJF9Y5njK4+zF3b/ruuW+R9Iy7f9fdfybpK5KuqXlMhbj7v0j6Yd3jKIu7v+juj678/yeSjklqdON1P+XllS/HV/4NHedqy1Gb2efN7HlJ10v6bF3jCOS3Jf1D3YOApFNv+Oe7vj6uhgeBNjOzLZJ2SHq43pEUZ2ZjZnZU0vclfdPdhz6mYIHazP7JzL6T8O8aSXL3T7v7Jkl3SPpoqHGUadAxrdzm05JO6NRxRS/LMTWcJXyv8WdwbWRmr5J0QNLv9px1N5K7L7v7dp06u36LmQ2dpgq2FZe7vyvjTf9O0r2S9oUaS1kGHZOZ/aakqyS90xtSoJ7jdWqq45I2dX19oaQXahoLUqzkcQ9IusPd7657PGVy9wUz+5akKyQNtQBcV9XH67u+vFrSU3WMo0xmdoWkT0q62t1fqXs8OO0RSa83s4vN7GxJH5B0T81jQpeVhbfbJR1z9y/UPZ4ymNmGTuWXmU1IepcKxLm6qj4OSNqqUxUF35P0EXefq3wgJTKzZySdI+l/Vr71UAsqWX5d0p9J2iBpQdJRd99V76jyM7P3SvqipDFJX3b3z9c8pELM7E5Jb9epFpr/LWmfu99e66AKMLNfk/Svkh7XqZggSb/v7vfVN6pizOyNkv5Gp/7m1km6y93/aOj7a8gZOgCMLK5MBIDIEagBIHIEagCIHIEaACJHoAaAyBGoASByBGoAiNz/A9sAXJTvp5M9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "m = 100 # 样本总数\n",
    "X = 6 * np.random.rand(m,1) -3\n",
    "y = 0.5 * X ** 2 + X + 2 + np.random.randn(m,1)\n",
    "plt.scatter(X,y)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "很清楚的看出，直线不能恰当的拟合这些数据。于是，我们使用 Scikit-Learning 的PolynomialFeatures类进行训练数据集的转换，让训练集中每个特征的平方（2 次多项式）作为新特征（在这种情况下，仅存在一个特征）："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "poly_features = PolynomialFeatures(degree = 2, include_bias = False)\n",
    "X_poly = poly_features.fit_transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.84309502])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.84309502,  0.71080922])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_poly[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "X_poly现在包含原始特征X并加上了这个特征的平方 X^2。现在你可以在这个扩展训练集上使用LinearRegression模型进行拟合"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr = LinearRegression()\n",
    "lr.fit(X_poly,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([1.96703475]), array([[0.97184305, 0.51554529]]))"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.intercept_,lr.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD7CAYAAABzGc+QAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU1dnA8d/JRhIChJCwJAESCDsEwqJgwIILS1lEhIIVtVVEqbbat1CxLmhbX1H6uhZLqVpsi0ARRCyIqEhVFiEYMCxhky0JOwQISch23j+SYEhmJrPcmbkz83w/Hz+SYebOuZPhuec+55znKK01QgghfE+QtxsghBDCORLAhRDCR0kAF0IIHyUBXAghfJQEcCGE8FESwIUQwkfVG8CVUu8opU4ppXbWeCxGKfWpUmp/1f+bureZQggharOnB74AGF7rsZnA51rrDsDnVT8LIYTwIGXPQh6lVBLwH61196qf9wKDtdbHlVKtgPVa6071HSc2NlYnJSW51GAhhAg027ZtO6O1jqv9eIiTx2uhtT4OUBXEm9vzoqSkJDIyMpx8SyGECExKqSOWHnf7IKZSaqpSKkMplXH69Gl3v50QQgQMZwP4yarUCVX/P2XtiVrr+VrrvlrrvnFxde4AhBBCOMnZAL4SuLfqz/cCHxrTHCGEEPaqNweulFoEDAZilVI5wCxgNvBvpdT9wFFggrMNKC0tJScnh+LiYmcPIUwgPDycxMREQkNDvd0UIQJGvQFca32nlb+62YgG5OTk0KhRI5KSklBKGXFI4WFaa86ePUtOTg7Jycnebo4QAcPrKzGLi4tp1qyZBG8fppSiWbNmchclhId5PYADErz9gPwOhfA8UwRwfxMVFQVAXl4e48ePt/ncV199lcLCQoeOv379ekaNGuV0+xy1YMECHnnkEQBWrFjB7t27PfbeQgjrJIDbqby83OHXxMfH8/7779t8jjMB3B5lZWWGHxMkgAsBsCIzl/TZ60ieuYr02etYkZlr9bnlFZpvj553SzsCPoAfPnyYzp07c++995Kamsr48eOvBtSkpCR+//vfM3DgQJYuXcrBgwcZPnw4ffr0YdCgQWRnZwNw6NAhBgwYQL9+/Xj66aevOXb37t2BygvA9OnT6dGjB6mpqbzxxhu8/vrr5OXlMWTIEIYMGQLA2rVrGTBgAL1792bChAkUFBQAsGbNGjp37szAgQNZvny5xXNZsGABEyZMYPTo0QwdOhSAOXPm0K9fP1JTU5k1axYAly9fZuTIkfTs2ZPu3buzZMmSq+d75swZADIyMhg8ePA1x9+4cSMrV65kxowZ9OrVi4MHD7r8+Qvha1Zk5vLE8ixy84vQQG5+EU8sz7IaxP+x6TDj3tzoliDu7FJ6v7J3717efvtt0tPTue+++3jzzTeZPn06UDk97uuvvwbg5ptvZt68eXTo0IFvvvmGX/ziF6xbt45HH32UadOmcc899zB37lyL7zF//nwOHTpEZmYmISEhnDt3jpiYGF5++WW++OILYmNjOXPmDH/84x/57LPPaNiwIS+++CIvv/wyv/3tb3nggQdYt24dKSkpTJw40eq5bNq0ie+++46YmBjWrl3L/v372bJlC1prxowZw5dffsnp06eJj49n1apVAFy4cMGuz+mGG25gzJgxjBo1qt7UkBD+as4neykqvfaOvKi0nDmf7GVsWsI1j+ecL2TOJ3sZ3CmOtNbRhrfFVAH8uY92sTvvoqHH7BrfmFmju9l8TuvWrUlPTwdg8uTJvP7661cDeHWwLCgoYOPGjUyY8MOU9ytXrgCwYcMGli1bBsDdd9/N448/Xuc9PvvsMx566CFCQio/8piYmDrP2bx5M7t3777alpKSEgYMGEB2djbJycl06NDhahvnz59v8VxuvfXWq8deu3Yta9euJS0t7eo57N+/n0GDBjF9+nQef/xxRo0axaBBg2x+PkKIH+TlF9n1uNaa332wEwU8f3sPtwz0myqAe0vtD7bmzw0bNgSgoqKC6Ohotm/fbtcxatNa2/WcW2+9lUWLFl3z+Pbt2+3+5Ve3t/p4TzzxBA8++GCd523bto3Vq1fzxBNPMHToUJ555hlCQkKoqKgAkCmBQlgRHx1BroUgHh8dwYrMXOZ8spe8/CKiI0M5X1jKc2O6kRAd4Za2mCqA19dTdpejR4+yadMmBgwYwKJFixg4cGCd5zRu3Jjk5GSWLl3KhAkT0Frz3Xff0bNnT9LT01m8eDGTJ09m4cKFFt9j6NChzJs3j8GDB1+TQmnUqBGXLl0iNjaW/v378/DDD3PgwAFSUlIoLCwkJyeHzp07c+jQIQ4ePEj79u3rBHhrhg0bxtNPP81dd91FVFQUubm5hIaGUlZWRkxMDJMnTyYqKooFCxYAlTnwbdu2MWLEiKt3FLVVt1eIQDVjWCeeWJ51TRpFAUnNIq55/HxhKUEKGoW7L8wG/CAmQJcuXXj33XdJTU3l3LlzTJs2zeLzFi5cyNtvv03Pnj3p1q0bH35YWQLmtddeY+7cufTr189qPnnKlCm0adOG1NRUevbsyXvvvQfA1KlTGTFiBEOGDCEuLo4FCxZw5513kpqaSv/+/cnOziY8PJz58+czcuRIBg4cSNu2be06r6FDh/LTn/6UAQMG0KNHD8aPH8+lS5fIysriuuuuo1evXjz//PM89dRTAMyaNYtHH32UQYMGERwcbPGYkyZNYs6cOaSlpckgpghIY9MSuKNPAjXviTWw4eC5OrnxCg3/t3af29pi14YORunbt6+uXQ98z549dOnSxWNtqO3w4cOMGjWKnTt31v9kYZO3f5dCeEr67HUW0yiWKODQ7JEuvZ9SapvWum/tx6UHLoQQDrI2kGlJvJvy3yABnKSkJOl9CyEcYm9QjggNZsawenebdFrAB3AhhHDUjGGdiAi1PE5ULVgpXhjXo87ccCOZYhaKPVPshLl5cixFCG+rDspzPtlrMRceERrs9uANJgjg4eHhnD17VkrK+rDqeuDh4eHebooQHjM2LeFqgL5vwVbWZVfuLJkQHcGMYZ3cHrzBBAE8MTGRnJwcZMNj31a9I48QgSbz6HnW7z3FpH6tmX1Hqkff2+sBPDQ0VHZxEUL4lOoVl7n5RYQEKRqHh/K7kZ6fQiuDmEII4YCa1QgByio0hSXlrNtzyuNtkQAuhBAOsFSNsKS8gjmf7PV4W7yeQhFCCG+rWYQqvp5BSHurEXqC9MCFEAHN0Q0arC3iceeKS2skgAshApqtDRosuaN33Z65u1dcWiMpFCFEQHMkJVJUUs7KHXnENAyjQUgQJy4U15tycScJ4EKIgGZtg4boyNA6j835ZC+Hzxby3gPXc0P7WE80zyZJoQghAtqMYZ0IDa67CryguOyaPPiWQ+f4+8ZD3DOgrSmCN0gAF0IEuLFpCTQMq5uMKK3QV/Pgl6+UMeP9HbRuGsnjwzt7uolWSQpFCBHwLhSVWny8Og/+x1V7OHqukMUP9KdhA/OETemBCyECnrUpgEFK8fyq3SzacpSpN7bj+nbNPNwy2ySACyECnrX63uVa87evDhEfHc7/3NrRCy2zTQK4ECLgjU1L4IVxPQi2UtK6rFzTIMT2Bg7e4FIAV0r9Wim1Sym1Uym1SCklBaGFED5pbFoCFVY2Jjl96YqHW2MfpwO4UioB+BXQV2vdHQgGJhnVMCGE8DRLc79tPe5trg6nhgARSqlSIBLIc71JQgjhHdZ2BrT0uCMFsNzF6R641joX+BNwFDgOXNBarzWqYUII4WnWphPWftzRAlju4koKpSlwG5AMxAMNlVKTLTxvqlIqQymVIdumCSHMLDaqgcXHa08zdLQAlru4Moh5C3BIa31aa10KLAduqP0krfV8rXVfrXXfuLg4F95OCCEctyIzl/TZ60ieuYr02eus9pKLSsoJDqo7C8VSpUGz1AR3JYAfBforpSJV5XbyNwN7jGmWEEK4zpFUxx9W7ebExWKm/ag9CdERKCp3mH9hXI86uW2z1AR3ehBTa/2NUup94FugDMgE5hvVMCGEcJWtVEfNoLxm53He++YoD97YjsdHdObxEbbrncwY1oknlmddc2xv1AR3aRaK1noWMMugtgghhKHsSXXk5Rfx+LIsUhOb8Juh9gXg6uDv7Vko5qnKIoQQBrNW67s61VFeoXlsyXbKyit4fVIaYSH2Z5XHpiV4ZROHmiSACyH80orMXC5fKavzeM1Ux9wvDrDl0Dn+b0JPkmIberqJLpMALoTwO9WDl7Xz300jQ5k1uhtj0xLYduQcr32+n7G94hlnYZ9LXyDFrIQQfsfS4CVAZFgIY9MSuFBUyq8WbSchOoI/jO2OslLEyuykBy6E8Du2Bi+11vzugyxOXixm6UMDaBRuzjon9pAALoTwO7Y2Kl6akcOq744zY1gn0to0dfjYZqiBUk1SKEIIv2Nto+JLRaU8uSKLG9o346EftXf4uGapgVJNArgQwu9Y26i4TFdOHXx1Yq9rls3bu9zeLDVQqkkKRQjhl6xVFqzQcPubG6+mQIZ0jmPZttyrgbm6Vw3USY2YpQZKNemBCyH8krW6JAquSYEs3HzU7l61WWqgVJMALoTwS9Y2Kq69N4OVPRws9qotHdMbNVCqSQAXQvilsWkJzBrdlZCqXHfLxo5t2WupV129+XF91Qo9RXLgQgi/pLVm0/dnqdCaRQ/0Z0D7ZqTPXmdxeqHi2p64rV61GWqgVJMeuBDCLy3acowPt+fx61s6MqB9M8B6CuSu/m1M06t2hPTAhRB+Z3feRZ79aBeDOsTy8JCUq4+bpQysUSSACyH8yoWiUqYt3EbTyFBendiLoFrbpJkpBeIqCeBCCL9RUaH5nyXbycsvYvHU/jSzskmxv5AcuBDCb7yx7gCfZ5/i6VFd6dM2xtvNcTsJ4EIIv/DF3lO8+vk+xqUlcHf/tt5ujkdIABdC+LyjZwt5dFEmnVs25vnbe/hsfW9HSQAXQvi0opJyHvrXNgDmTe5NRFjd1Zf+SgYxhRA+S2vNkyuy2HPiIu/c24+2zXxvX0tXSAAXQvic6k0VqldVDu/WkiGdm3u5VZ4nKRQhhE+pualCtfV7T3ltUwVvkgAuhPApljZVKC6r8NqmCt4kKRQhhE+xVIwK7NtUwUz7WRpBArgQwmdorYkIDa7TA4f6N1WoTr3Ys/OOr5AUihDCZ7y5/iBFpeVXa3xXs2dTBbPtZ2kE6YELISwyW7rh090n+dPavYzpGc+QTnH8ae0+h9rmSurFrCSACyHqMFu6Yd/JSzy2OJPu8U14aXwq4aHB3N470e7Xr8jMrbNpQzVv7WdpBJdSKEqpaKXU+0qpbKXUHqXUAKMaJoTwHjOlG85fLmHKuxlEhIUw/54+hFvY57I+cz7ZazF4K/DafpZGcLUH/hqwRms9XikVBkQa0CYhhJdZSyt4Ot1QWl7Bw+99y4kLxSya2p9WTZzrLVtrt8Z3BzDBhR64UqoxcCPwNoDWukRrnW9Uw4QQ3mMtreDJdIPWmqdX7GTjwbP877ge9Gnb1OljWWt3gg+nT8C1FEo74DTwd6VUplLqLaVUYBUiEMJPWds70pPphr9++T2Ltx7j4SHtGd/H/ny3JWY4H3dwJYCHAL2Bv2it04DLwMzaT1JKTVVKZSilMk6fPu3C2wkhPGVsWgIvjOvhtY1+P846zuyPsxmV2orf3Op6kPX2+biL0tpSat+OFyrVEtistU6q+nkQMFNrPdLaa/r27aszMjKcej8hRGDYfiyfiX/dRNf4xix6oL9Tg5b+Rim1TWvdt/bjTvfAtdYngGNKqerL483AbmePJ4QQOecLmfJuBs0bN+Bv9/SV4F0PV2eh/BJYWDUD5Xvg5643SQgRiC4Wl3L/ggyulJWzeOr1xPr5hsRGcCmAa623A3W69UII4Yiy8goeeS+Tg6cLePe+60hp3sjbTfIJshJTCOFVWmue/nAnX+47zYt39CA9JdbbTfIZUsxKCOFVr32+n0VbjvHIkBQm9mvj7eb4FOmBCyHcpr6CWIu2HOXVz/YzoU8ivxna0eXjBRoJ4EIIt6ivINZnu0/y5AdZDO4Ux/+O64FSytbhTFdgywwkgAsh3MJWQaw2zSJ5ZNG3dE9owohuLRk8Z329vWpbx5MALoQQBrJWQCo3v4j7F2ylZeNwJvRJ5NmPdtvVqzZLgS0zkUFMIYRbWCsgFRykCA5SvHvfdcz77/d2l601Q4Ets5EALoRwyIrMXNJnryN55irSZ69jRWauxedZKiClqAzg7/ysH22bNXSoV+2vBalcISkUIYTdHBlIrP55zid7yc0vIiw4iHKt+ds9fUlNjAYqe8+Wtjqz1KuueTyZhVLJ6WJWzpBiVkL4tvTZ6ywG3KaRoWQ+M9Tia0rKKpjyjwy+3n+aN+7szcjUVlf/rvYFASp71f5QKdBIhhezEkIEHmspj/OFpRZTKeUVmseWZPLlvtPMHpd6TfAG/y3z6imSQhFC2M1aygOoM51Pa80Ty79jddYJnhrZhZ/0a23xdWPTEiRgO0l64EIIu9kaMKzZO9da88dVe/h3Rg6/urkDUwa180TzAo4EcCGE3camJRAdEWrx72oOPL6x7gBvf32In92QxK9v6eCp5gUcCeBCCIc8O6abzel8b331PS9/uo9xvRN4ZlTXepfIC+dJDlwI4RBb0/kWbDjEH1ftYUT3lrx0RypBQRK83Ul64EIIQ/xz8xGe/Wg3w7q14PU70wgJlvDibtIDF0I4xNJinhnv76C0XHNLl+a8cWdvQiV4e4R8ykIIh1iqClharmkQEsTcu3oTFiJhxVPkkxZCOMTaYp4rZRU0CJFd5D0pIFMosquHEM6ztpgnIYCrAnpLwPXAq/N3uflFaH4oxmOtopoQ4lozhnUiNPja2SWBXhXQWwIugNva1UMIUb+CK2VXc97wQ/0SwK4ys8I4AZdCMXpXD0nHiEDy9teH+MN/dnNz5+bMvas34VULelZk5jJj6Q5KKyqrm+bmFzFj6Q4gcPer9ISAC+CO1B+uj2yyGth8/eLtaPvfXH+Al9bsZUT3lrw2Ke2a2SbPrtx1NXhXK63QPLtyl099Jr4m4FIoRu7qIemYwOXrYymOtF9rzSuf7uOlNXu5rVc8b9yZVmeqYH5RqcX3sfa4MEbABXAj6w/LJquBy9cv3va2X2vN7DXZvPb5fib0SeTln/SSFZYmEnApFDCu/rCR6RjhW7xx8TYyZWNP+8srNE9+kMXirceY3L8Nvx/T3Wptk6aRoZwvrNvbbhppuXKhMIZcSl0gm6wGLk/tkF69gXDSzFX8esl2w1I29bW/uLSchxd+y+Ktx/jlTSn84TbrwRtg1uhudaYWhgYrZo3u5lT7hH38sgfuqcEl2WQ1cM0Y1sniXo5GXrxrD5LX3r22OuVR+/tmz/ffVvsLrpQx9R8ZbDx4lqdHdeX+gcn1Hlv+LXiHy5saK6WCgQwgV2s9ytZz3bGpce0v1JDOcSzbliubpAq3c3dHwdoGwjUp4NDskVfb8+zKXXUGDmt+/2u2OToyFK3hQlHp1fYP6hDLz/6+ld3HLzJnfCrjeidec76yAbF3WNvU2Ige+KPAHqCxAcdyiKVpfAs3H7W7pyKEK9y9l6M9+fTqlIel4Fqt5uBkzeecLywlIjSYVyb2YmxaArn5RUyYt4nc/CL+dk8fburc4prj2Br4lH9b3uFSDlwplQiMBN4ypjmOsfSFsnY/ITNDhK+pL59eM2Vj6d9CTXn5RTYD8J7jF7njzY2cKbjCv6ZcXyd4Vx/D2rGFd7g6iPkq8FugwoC2OMyRL47MDBG+xtIgefUwYe3pr/X9W4iPjrD6nOqeN8CSBwfQLynG6jEceVy4n9MBXCk1Cjiltd5Wz/OmKqUylFIZp0+fdvbtLLL2xak9Vi4zQ4QvsrRm4ZWJvTg8eyQbZt50TdrCVhCt/v7bek5i0wg+ePgGurSyngmVWVfm4/QgplLqBeBuoAwIpzIHvlxrPdnaa4wexLQ2qHJHnwS+yD4to+EiYFjLgTeNDGXW6G5XBzAtPadTy0a8/9AAGoXXP2fb18sH+Cprg5guz0KpOvhgYLoZZqHIF8p3ye/SNfZ8fisyc3lpTTZ5F4oB6J8cwz+nXC9boJmcO2eheJW7ZwIIz5DCYK5fwOz5t/CjjnEs3nqUvAvFTB/akYeHpKCU7Bzvqwy57Gqt19fX+xbCFl+vLeIqTxTH2n/yEmPf3MC3R/J5ZWJPHrmpgwRvH+cT90278y6SfeKit5sh3CjQp6i5+wL2+Z6T3P7mRi5fKWfR1P7cnpZY/4uE6Zk+gGutefrDndw+dyP/+S7P280RbhLoU9TcdQHTWvOX9QeZ8o8MkmIjWflIOn3aNnXpmMI8TB/AlVL85a7edI1vzCPvZTL742zKK1wfeBXmEuhT1NxxASsuLed//r2DF9dk8+MerVj64A0Bc0EMFD4xiNm8cTiLHujPsx/tYt5/D7L7+EVen9SL6MgwbzfNFPxh9kagF0NypjiWrd97Xn4R0xZ+y45j+fzm1o48cpMMVvojQ6YR2suIaYSLthzlmQ930qpJBPPv6UPnlh4vwWIqUmDIf1gLyJYeB6z+3uMaNeCXizK5UlrO//2kF8O7t/TWKQmDuHUeuL2Mmge+7ch5pv1rG5eKy5gzIZVRqfEGtM43WatYlxAdwYaZN3mhRcJI1i7Q4aFBFjdQaBweQsGVMtrHRTHv7j60j4vyZHOFm1gL4KbPgVvSp21T/vPLgVfz4rM+3MmVMuuFfPxZoM/e8HfWZqdYCt4AF4vLGJkaz4qH0yV4BwCfDODwQ158ysBk3t10hAnzNnH0bKG3m+VxgT57w985eiFuEhHK65N60bCBTwxvCRf5bAAHCAsJ4qlRXZl/dx8On7nMyDe+Ys3OEx57/+rtrpJnriJ99jrDFl04ctxAn73h76xdiKMjQgmvtTN8WHAQz43pJoOVAcSnA3i1od1asupXg2gX25CH/rWN5z7aRUmZeyvcumvlnKPHtVSxTgYw/YetC3THlo2uPtaycTgvjU+V33uA8clBTGtKyip44eM9/H3DYVITm/DqxF60c1Me0F2DhzIoKWqrPQvljt4JvL8th1OXrvDb4Z2YMrCdzQ2Hhe/z22JWNYWFBDFrdDeuT27G48u+Y+TrX/PM6K5M6tfa8NtKdw0eyqCkfzBybn51karyCs2f1x3gtc/30TomkmXTbqBn62iDWy58iV8F8GrDu7ekV+topi/dwRPLs1iXfYrZ43rQLKqBYe8RHx1hsafs6uChu44rPKe+yorOBPdj5wr5zdIdbDl0jrG94vnD2O521e8W/s0vcuCWtGwSzj/uu46nRnbhv3tPM/y1r1i/95Rhx3fX4KEMSvo+W4WpHB3j0FqzeMtRhr/6JbvzLvKnCT15ZWIvCd4C8OMADhAUpJgyqB0fPpJOTGQYP/v7Vp75cCeXr5S5fGx3DR7KoKTvs5UGc6Tq4KmLxdz/bgYzl2eRmhjNmscGMb5PoswyEVf51SCmLcVV/0je2XCIhOgIXrwjlfSUWK+0RXiXu2vH2BqIzqvqedemgEOzR179+aMdeTz94U6KSsqZOaIz9w5IkoHKAOZXKzGdER4azNOjurL0wQGEBgdx11vf8MTyLC4VW17RJszFqDn3ntg4wVYarL6FV6cuFfPwwm/55aJM2jZryOpHB/Hz9GQJ3sKigOmB11RcWs7Ln+7jra++p0XjcF4Y14PBnZp7u1nCCiMLdnlqmqatwlSWzuV/b+9OSXkFz6/aQ3FpBY/e0oEHb2xHiOxVKfCzYlZGyTx6nhnvf8eBUwWMS0vgdyO7EGvgTBVhDCODbvLMVXalMNypdnD/2Q1JfJ59ks3fn+O65BheGNfD0Dom/lBuONAFxDxwa6x9gdPaVBbF+vO6A/z1y4N8tuckj4/ozJ392sgtq4kYOTfe1WmaRgTD6nndy7bl8NxHu3h+9R4UMLFva14Y18PQ755sFu3f/P7+rL6cZ3hoMNOHdeLjRwfRNb4xT36wk3F/2cjO3Avebbid3FWPxUyMLNjlyjRNI/PnL63JZvrSHVwsrpwRpYGVO/JYucPYbQMDfbNof+f3AdzeL3BK80YseqA/r0zsSc75Qsb8+Wue+2gXF008yOmJATkzMHJuvCvTNI0Ihrn5Rfxi4TbeXH+wTirHHYFVVvb6N79PoTjyBVZKcXtaIjd1asGctdks2HiYldvz+PWtHZnUr7XpBpRsBRR7b499IT9q9HZr1SkMR7kSDItLy/nbl98zd/0Bp97DWbKy17/5fQB35gvcJDKUP47twaR+bfj9f3bz1Iqd/HPTEZ4a1YVBHeLc2VyHuNq78qX8qLNB10jOfJe01qzOOsGLa7I5eq6QH/doyZMju/KTeZs8Elid2WtT+A5zdSndwJnb7+q88ug3vibnXCE/T0+isLSMu9/ewv0LtnLwdIEhbXMmf13zNUFWVuTZGwQkP+oYR79LWw6d4/Y3N/Lwe98SGRbMv+6/njfv6kNC1R2EJ0omyMpe/+b3PXBHb79r90rzLhSzeMsx/nBbN85eLuGNdQcY+sqX/KRva351cwqtmjjXY3Km91v7NeUWpoA6EgTclR/1hbSMM+z9Lh04VcDsj7P5bM/Jq3W67+idSHCN2SVGp4Xqa7c/fP6iroCeB25JfXOOzxRc4c/rDrDwmyMopbinf1umDW7vcKVDZ+Y2W3tNsFJUaO1wEHDHohYjF91YO75ZLw455wuZ+8UB/p2RQ0RoMNMGt+e+9GQiwoLrf7EQNgT0PHBH1NcrjY1qwLNjunH/wGRe/3w/72w4xKItR7l/YDJTbmxHYzurxDnT+7X2dxVaO7UIxR35UXsGVp0NwmbN2efmFzH3iwMszTiGQnF3/7b88qYUQ8sXC2GJBPBa7B2oah0TyZwJPXnwR+155dN9vL7uAH/feJif3ZDEz9OTiWkYZsj7uPoaW9xxG1/fhcmVIGzErBsjHb9QGbiXbD0GwMR+rfnF4BSZ4SE8RgJ4LY72SlOaRzH3rt5My73An9cd4I11B3jrq0P89Po2TL2xHS0ahzv1PpZ6qe7oMRudH63vIuNKEHbkrsWdqfo23vUAAA+eSURBVJaDpwt466vvWbYtF41mQt/WPDwkhQQJ3MLDJIDX4myvtHtCE+bd3Yf9Jy/xl/UHWbDxMP/cdITxfROZMjC5zt6ctt7HWi/1hXE9eGFcD1PkgGsHyCGd4/gi+zS5+UUouGaRSs2LjCsDp/begbgr1bLtyHn++t+DfLrnJKHBQUzom8i0we3JOHyen8zb5PXfiQg8Tg9iKqVaA/8AWgIVwHyt9Wu2XuMLg5hGOXaukHn/PcjSjBxKyisY0imOn6cnM6hDbL0F+c2+sbGlgcraqoN4Qq2A5sq52TtAauTnV16h+WzPSd766nu2Hj5Pk4hQ7h3QlntuSCI2qoHbB22FAPcMYpYBv9Faf6uUagRsU0p9qrXe7cIx/UbrmEiev70Hj93SkYXfHOFfm49yzztbSGkexc9uSGJc7wQiwyx//GZf/mwpDVJbdfCuHTBdSQPZe3dkxOd3puAKS7Ye471vjpKbX0RCdASzRnflJ31b07DBD783s+XlRWBxOoBrrY8Dx6v+fEkptQdIACSA1xDXqAGP3dKRaYPbs+q747yz4RBPrdjJi2uyGdsrgYn9WtM9ock1rzH78md7A6Gl57k6cGpPzt7Zz09rzbdHz/PPTUdYnXWCkvIK0lOa8fSortzSpbnFUgpmv9gK/2ZIDlwplQSkAd8YcTx/1CAkmHG9E7k9LYGMI+dZuPkISzKO8c/NR+ie0JhJ/dowplc8jcNDTb/82VqAtPQ8S9y9sMTRz+/EhWI+yMxl2bc5HDhVQKMGIfz0+jZM7t+WlOa263Kb/WIr/JvLC3mUUlHAf4HntdbLLfz9VGAqQJs2bfocOXLEpffzJxcKS1mxPZdFW46SfeIS4aFBDO3aktE947lQWMIrn+035cCYPTlwb+eB65uFUlxaztrdJ1m2LYev9p+mQkPftk0Z3yeR0T3jr0mT1Pc+tT8La/l/IZzllh15lFKhwH+AT7TWL9f3fDMMYppxJZ/WmqzcCyzZeozVWcc5X1hK4/AQRnRvxZhe8fRv1+yaZdhm8NSKLBZ9c4xyrQlWiv7tmnL4bJHd5Qq88TsoLi1n/d5TrM46wed7TnK5pJz4JuHc0SeRcb0TSY5t6NRxq8/H2gyc6guZGb97wjcYHsBV5VSKd4FzWuvH7HmNtwO4L8wYKC2v4Ov9Z1i5I4+1u05wuaSc2KgwburcnFu6tGBgh1irg5+e4srnaNTvwN5geKm4lC/3nWH1zuN8kX2KwpJyYhqGMaxbC0alxjOgXTPDdsCxNfvFWlrHTN89YV7uCOADga+ALCqnEQL8Tmu92tprvB3AjZpe5qmeVFFJOeuyT7Fm1wnWZ5/i0pUyGoQEMTAllpu7tODGjrEkNo00/H3r48rnaMTvwNZF4LZe8ew7WcD6vaf4Yu8pMg6fp6xCExsVxrBuLflxj1ZcnxzjltrutvbbtJYrN8vUUGFuhk8j1Fp/TeV302cYMWPAk/U4IsKCGZnaipGprSgpq2Dr4XN8tuckn+4+yefZpwBo2yySG9rHkp7SjAHtmnmk/oYrn6MRvwNrU/d+90EWL63JJu9CMQCdWzbigRvbMbhjHH2TYtyehrI1oCmzVYQ7BNRKTEdnDFjqaXtr3m9YSBDpKbGkp8TyzKiu7D9VwIYDZ9hw4Cz/2ZHHoi1HAejYIorebZqS1iaatDZNSYmLYuWOPEPvGFyZeeHqrI2SsgqrM2AKS8oZ1CGWX93cgR91inO61K+zbM1+qc6R1yazVYQrAiqAOzK9zFpP29rMC0/2pJRSdGzRiI4tGvHz9GTKyivIyr3AxoNn2Xr4HB/vPMHiqgJL4aFBlJRVUFF1b5+bX8TMZd8Bzt8xuDLN0ZHXXiou5cCpAvadvERW7gWyci6w58Qlq8eObxLOX++uc5fpMfXNcTfz1FDhmwKuHri9+WtbtbctbaRgplym1ppDZy6TeTSfZz7cyeUS69P9GoYFM7RbS0b3bEWbmEgSoiPtql/tyjhAzVkbLRuHM7l/Gzq0aERefhFHzhZy4FQB+09d4uTFK1df06hBCN0SGpOaGE1xaTlLth7jSlnF1b/3hQFBmYUinOWWaYSOMkMAt5e1ASmoDBaemkXhKlvnYU1UgxBio8KIa9SA2KgGNIsKI6pBKFENgmnYIISGYSE0bBBCaLAiSCmCgirvCoKqNpa4UlpOcWkFxaXlFJeWU1RaQX5hCecul3C+sJTzhSWcv1zCiYvFFNa6uESGBZPSPIqUuChSWlT+v0OLRrSNibw6W2RFZi7PrtxFflEpAE0jQ5k1upsEQ+G3ZEMHB9maNVCd03Q2+HpyINTeVZNxUQ14alQXcvOLOHOphNMFVzhz6Qr7TxWw+fsrFFwpo7Tc+Yt9WEgQzRqGER0ZRkzDULrEN2ZI5+bER0eQEB1e9f8IYhqG2Sz2ZWkGSnFphdXnC+HPJIBbYStX6+pScE8OhFo6D0vOFFzhtl6237ukrILLV8oouFLG5ZIyyso1WlfuCFT5HygF4SHBhIcGER4aTHhoMBGhlT/XV4XRHt4YRJbUhzArCeBWuHPTWU9OKat9HkFWcvj2zIYICwkiLCSMpvXsNuROnp6OZ9Zt3IQACeA2uavokqcLINU8D2uLYMwwG8Kenq6zn52zvWgpFyvMzPjlaKJeM4Z1IiL02pkengqiY9MSeGFcDxKiI1BU5vTNMHuj+sKSm1+E5oee7orM3Gue58xnZ++xLZEFOMLMpAfuBZbSM0M6xzHnk738esl2t+dZ3V3O1Rn29nSdSW1ZO/azK3e5rba4EJ4gAdxLbKU13JlnNeuAnCM9XUcvQNaOnV9UyorMXJvHMnttdhHYJIViArZ6n0ZyJZXgbtZ6tEb0dG0do77P2KwpJyFAeuCm4Kk8q5kH5NzZ050xrBOPLdlu8e/s+YzNmHISAqQHbgru7H3WZOYBOXf2dMemJdA0MtTi30kuW/gy6YGbgKfyrGYfkHNnT3fW6G6SyxZ+RwK4Cbhz0VBNjlZjNONgp7M89RkL4UlSzCrA2BOYfWHrOSECiRSzEoB9aQozD3YKIX4gg5iiDmuDmrn5RaaYciiEqCQ9cD/nTC7bVglaKeQkhHlIAPdj1lZ4Zhw5xxfZp60GdVslaB1JpRg9EOpvA6tCuEoGMf2YtW3hFFyzS4+lAcoVmblWF78o4NDskTbf2+iB0BWZucxYuoPSimu/r9ERoTw7RnbjEf7N2iCm5MD9mLVcdu1LtqVl+2PTEkhwYYGR0eUBnl25q07whsp6JmYpByCEp0kA92OOLNCxFOxdKXtr9KrP6v0vLXFH3RghfIEEcD9mKQBb29TMUrB3ZXm7p8oDVDNDOQAhPE0GMX2As4N31uqOL9uWa/eScmeXtxtdHqBpZCjnC633ws1SDkAIT5JBTJNzx6pIT83mMPJ9nlqRxb82H7X4d9WDsgkyM0X4KWuDmBLAPcTZYGZtJklCdAQbZt7kjqaajqWLWDV7ZtQI4etkFooXyZ6MrrE0owUgWCm7ZtQI4a8kgHuAK1PqPD0YaEbWLlblVu4eA+niJgKbSwFcKTVcKbVXKXVAKTXTqEb5G1d60d7cwd4srF2sgpXlOTWBdHETgc3pAK6UCgbmAiOArsCdSqmuRjXMn7jSi5Y9Ga1fxO68vnXAX9xEYHNlGuF1wAGt9fcASqnFwG3AbiMa5k9cnVIX6Hsy2tqMoW/bGKmPIgKWKwE8AThW4+cc4HrXmuOfZDcY11m7iAX6xU0ENlcCuKUEZJ1RJaXUVGAqQJs2bVx4O98mgUYIYTRXAngO0LrGz4lAXu0naa3nA/Ohch64C+8nhMdJCVthZq4E8K1AB6VUMpALTAJ+akirhDABa/XUQTa0EObg9CwUrXUZ8AjwCbAH+LfWepdRDRPC24wuiSuE0VwqZqW1Xg2sNqgtwkBy6+86WQUrzE5WYvohV5buix/IKlhhdhLA/ZDc+htDVsEKs5N64H5Ibv2NIfP3hdlJAPdD8dERFkvQyq2/42T+vjAzSaH4Ibn1FyIwSA/cD8mtvxCBQQK4n5JbfyH8n6RQhBDCR0kAF0IIHyUBXAghfJQEcCGE8FESwIUQwkdJABdCCB8lAVwIIXyU0tpzm+QopU4DR5x8eSxwxsDmeJOci/n4y3mAnIsZuXoebbXWcbUf9GgAd4VSKkNr3dfb7TCCnIv5+Mt5gJyLGbnrPCSFIoQQPkoCuBBC+ChfCuDzvd0AA8m5mI+/nAfIuZiRW87DZ3LgQgghruVLPXAhhBA1+FQAV0r9QSn1nVJqu1JqrVIq3tttcpZSao5SKrvqfD5QSkV7u03OUEpNUErtUkpVKKV8craAUmq4UmqvUuqAUmqmt9vjLKXUO0qpU0qpnd5uiyuUUq2VUl8opfZUfbce9XabnKWUCldKbVFK7ag6l+cMPb4vpVCUUo211her/vwroKvW+iEvN8spSqmhwDqtdZlS6kUArfXjXm6Ww5RSXYAK4K/AdK11hpeb5BClVDCwD7gVyAG2AndqrXd7tWFOUErdCBQA/9Bad/d2e5yllGoFtNJaf6uUagRsA8b66O9EAQ211gVKqVDga+BRrfVmI47vUz3w6uBdpSHgO1efWrTWa7XWZVU/bgYSvdkeZ2mt92itfXm7++uAA1rr77XWJcBi4DYvt8kpWusvgXPeboertNbHtdbfVv35ErAH8MndSXSlgqofQ6v+Myxu+VQAB1BKPa+UOgbcBTzj7fYY5D7gY283IkAlAMdq/JyDjwYLf6SUSgLSgG+82xLnKaWClVLbgVPAp1prw87FdAFcKfWZUmqnhf9uA9BaP6m1bg0sBB7xbmttq+9cqp7zJFBG5fmYkj3n4cOUhcd89s7OnyilooBlwGO17r59ita6XGvdi8q77OuUUoalt0y3J6bW+hY7n/oesAqY5cbmuKS+c1FK3QuMAm7WJh6McOB34otygNY1fk4E8rzUFlGlKl+8DFiotV7u7fYYQWudr5RaDwwHDBloNl0P3BalVIcaP44Bsr3VFlcppYYDjwNjtNaF3m5PANsKdFBKJSulwoBJwEovtymgVQ38vQ3s0Vq/7O32uEIpFVc9w0wpFQHcgoFxy9dmoSwDOlE56+EI8JDWOte7rXKOUuoA0AA4W/XQZl+cUaOUuh14A4gD8oHtWuth3m2VY5RSPwZeBYKBd7TWz3u5SU5RSi0CBlNZ+e4kMEtr/bZXG+UEpdRA4Csgi8p/6wC/01qv9l6rnKOUSgXepfK7FQT8W2v9e8OO70sBXAghxA98KoUihBDiBxLAhRDCR0kAF0IIHyUBXAghfJQEcCGE8FESwIUQwkdJABdCCB8lAVwIIXzU/wPyr+dRZu2i/QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_plt = np.array([i /1000 for i in range(-3000,3000)])\n",
    "X_plt_poly = poly_features.fit_transform(X_plt.reshape(-1,1))\n",
    "y_predict = lr.predict(X_plt_poly)\n",
    "plt.plot(X_plt,y_predict, label = \"predicted result\")\n",
    "plt.scatter(X,y)\n",
    "plt.legend(loc = 'best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 学习曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "def plot_learning_curves(model, X, y):\n",
    "    X_train,X_val,y_train,y_val = train_test_split(X, y, test_size = 0.2)\n",
    "    train_errors, val_errors = [],[]\n",
    "    for m in range(1, len(X_train)):\n",
    "        model.fit(X_train[:m],y_train[:m])\n",
    "        y_train_predict = model.predict(X_train[:m])\n",
    "        y_val_predict = model.predict(X_val)\n",
    "        train_errors.append(mean_squared_error(y_train_predict, y_train[:m]))\n",
    "        val_errors.append(mean_squared_error(y_val_predict, y_val))\n",
    "    plt.plot(np.sqrt(train_errors),\"r-+\",linewidth = 2, label = \"train\")\n",
    "    plt.plot(np.sqrt(val_errors),\"b-\", linewidth = 3, label = \"val\")\n",
    "    plt.legend(loc= \"best\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU5bkH8N+ThYSELYQEkSgJCKiAgkRE7bXiVkTcWtvi8rGXWuit1O3Wa9XaS7yttr2t1lq1XmqtrQviUqpStVYKRSxgE6SVfRMkLJIAgYAkJJnn/vHMYWaSSXImzGTeJL/v53M+M2fmzMwz22/e8573nBFVBRERuSsl2QUQEVHLGNRERI5jUBMROY5BTUTkOAY1EZHj0hJxp/369dPCwsJE3DURUadUVlZWqap50a5LSFAXFhaitLQ0EXdNRNQpicjW5q5j1wcRkeMY1EREjms1qEVkuIisCJsOiMjt7VEcERH56KNW1XUARgOAiKQC2A5gboLrIqIupq6uDuXl5aipqUl2KQmVmZmJgoICpKen+75NrBsTLwSwSVWb7fQmImqL8vJy9OzZE4WFhRCRZJeTEKqKPXv2oLy8HEVFRb5vF2sf9RQAs6NdISLTRaRUREorKipivFsi6upqamqQm5vbaUMaAEQEubm5Ma81+A5qEekG4AoAL0e7XlVnqWqxqhbn5UUdCtiiVauAhQuBt98GKitjvjkRdQKdOaQ9bXmOsbSoLwWwXFU/jflRfLjtNmDCBODSS4EPP0zEIxARdUyxBPW1aKbbIx4yM0PnO/m2BCJyUFVVFZ544omYbzdp0iRUVVUloKIQX0EtIlkALgbwh0QVwqAmojYpKYnL3TQX1A0NDS3e7s0330SfPn3iUkNzfAW1qn6mqrmquj9RhTCoiahN7r8/Lndz9913Y9OmTRg9ejTOPPNMTJgwAddddx1GjRoFALjqqqswduxYjBgxArNmzTp6u8LCQlRWVmLLli045ZRTMG3aNIwYMQKXXHIJDh8+HJfanNkzkUFNREeJ+J9iWb4FP/7xjzFkyBCsWLECP/3pT/HBBx/ggQcewOrVqwEATz/9NMrKylBaWopHH30Ue/bsaXIfGzZswIwZM7Bq1Sr06dMHr776alxejoQclKktwoO6tjZ5dRARAcC4ceMixjo/+uijmDvX9vXbtm0bNmzYgNzc3IjbFBUVYfTo0QCAsWPHYsuWLXGpxcmgZouaqIuL5U+3RWJb3qfs7Oyj5xcuXIh3330XS5YsQVZWFs4///yoY6EzMjKOnk9NTe18XR9hz49BTUTtrmfPnqiuro563f79+5GTk4OsrCysXbsWS5cubdfa2KImoo5t5sy43E1ubi7OPfdcjBw5Et27d0f//v2PXjdx4kQ8+eSTOO200zB8+HCMHz8+Lo/pF4OaiDq2OA3PA4AXXngh6uUZGRl46623ol7n9UP369cPK1euPHr5nXfeGbe6nOn6YFATEUXHoCYichyDmojIcQxqIiLHMaiJiBzHoCYicpyTQc1dyInIdT169Gi3x3IyqNmiJiIKcWaHF+5CTkTJ9N3vfheDBg3CzTffDAAoKSmBiGDRokXYt28f6urq8MMf/hBXXnllu9fGFjUROSeWo5zGOjVnypQpmDNnztH5l156CVOnTsXcuXOxfPlyLFiwAN/5znegCTgAVGucaVEzqIkomcaMGYPdu3djx44dqKioQE5ODgYMGIA77rgDixYtQkpKCrZv345PP/0Uxx13XLvWxqAmIgq65ppr8Morr2DXrl2YMmUKnn/+eVRUVKCsrAzp6ekoLCyMenjTRGNQE5FzktC7AMC6P6ZNm4bKykr87W9/w0svvYT8/Hykp6djwYIF2Lp1a1Lq8vvntn1E5BURWSsia0Tk7HgXwqAmomQbMWIEqqurMXDgQAwYMADXX389SktLUVxcjOeffx4nn3xyUury26L+BYC3VfUaEekGICvehaSnh/6oob7epjRn2vtE1FV89NFHR8/369cPS5YsibrcwYMH26uk1lvUItILwHkAfgMAqnpEVaviXYgId3ohIorGT9fHYAAVAH4rIh+KyFMikt14IRGZLiKlIlJaUVHRpmLY/UFE1JSfoE4DcAaAX6nqGACHANzdeCFVnaWqxapanJeX16Zi2KIm6tqSMUa5vbXlOfoJ6nIA5aq6LDj/Ciy4444taqKuKzMzE3v27OnUYa2q2LNnDzLDw86HVjfXqeouEdkmIsNVdR2ACwGsbmOdLeJu5ERdV0FBAcrLy9HWrtOOIjMzEwUFBTHdxu+4ilsAPB8c8bEZwNQYa/OFLWqiris9PR1FRUXJLsNJvoJaVVcAKE5wLQxqIqIonDkoE8CgJiKKhkFNROQ4BjURkeMY1EREjmNQExE5jkFNROQ4Z4Oau5ATERlng5otaiIi41RQcxdyIqKmnApqtqiJiJpiUBMROY5BTUTkOAY1EZHjGNRERI5jUBMROY5BTUTkOAY1EZHjnA1q7kJORGScDWq2qImIjFNBzV3IiYia8vXntiKyBUA1gAYA9aqakD+6ZYuaiKgpX0EdNEFVKxNWCRjURETRONX1waAmImrKb1ArgHdEpExEpkdbQESmi0ipiJRWVFS0qRgGNRFRU36D+lxVPQPApQBmiMh5jRdQ1VmqWqyqxXl5eW0qJnxjYm0toNqmuyEi6lR8BbWq7gie7gYwF8C4hBSTAnTrFprnWGoiIh9BLSLZItLTOw/gEgArE1UQuz+IiCL5GfXRH8BcEfGWf0FV305UQZmZwIEDdp5BTUTkI6hVdTOA09uhFgDcjZyIqDGnhucB7PogImqMQU1E5DjngprH+yAiiuRcULNFTUQUiUFNROQ4BjURkeMY1EREjmNQExE5jkFNROQ4BjURkeOcDmruQk5E5HhQs0VNRMSgJiJynnNBzV3IiYgiORfUbFETEUViUBMROY5BTUTkOAY1EZHjGNRERI5jUBMROc53UItIqoh8KCLzElkQ90wkIooUS4v6NgBrElWIhy1qIqJIvoJaRAoAXAbgqcSWw6AmImrMb4v6EQB3AQg0t4CITBeRUhEpraioaHNBDGoiokitBrWITAawW1XLWlpOVWeparGqFufl5bW5IO5CTkQUyU+L+lwAV4jIFgAvArhARJ5LVEFsURMRRWo1qFX1HlUtUNVCAFMA/FVVb0hUQQxqIqJIzo+jVk1eLURELogpqFV1oapOTlQxAJCWBqSm2vlAAKivT+SjERG5z7kWNcDuDyKicAxqIiLHOR/U3I2ciLo654OaLWoi6uoY1EREjmNQExE5zsmg5m7kREQhTgY1W9RERCEMaiIixzGoiYgcx6AmInIcg5qIyHEMaiIixzkf1NyFnIi6OueDmi1qIurqGNRERI5jUBMROc7JoOYu5EREIU4GNVvUREQhDGoiIse1GtQikikiH4jIP0VklYjcn+iiGNRERCFpPpapBXCBqh4UkXQAi0XkLVVdmqiiGNRERCGtBrWqKoCDwdn04KSJLIpBTUQU4quPWkRSRWQFgN0A/qKqy6IsM11ESkWktKKi4piKYlATEYX4CmpVbVDV0QAKAIwTkZFRlpmlqsWqWpyXl3dMRXEXciKikJhGfahqFYCFACYmpJogtqiJiEL8jPrIE5E+wfPdAVwEYG0ii2JQExGF+Bn1MQDA70QkFRbsL6nqvEQWxaAmIgrxM+rjXwDGtEMtR3EXciKiEO6ZSETkOAY1EZHjnAzqbt1C5+vqgIaG5NVCRJRsTga1CMdSExF5nAxqgN0fREQeBjURkeM6RFCz64OIurIOEdRsURNRV+ZsUGdnh85v3Zq8OoiIks3ZoP7c50LnX3opeXUQESWbs0F93XWh86++yu4PIuq6nA3qM88Ehgyx8wcOAG++mdx6iIiSxdmgFolsVT//fPJqISJKJmeDGogM6j/9CaiqSl4tRETJ4nRQn3wycMYZdr62Fpg7N7n1EBElg9NBDUS2ql94IXl1EBEli/NBPWWK9VcDwF//Cuzcmdx6iIjam/NBPXAgcP75dj4QAObMSWo5RETtzvmgBtj9QURdm59/IT9BRBaIyBoRWSUit7VHYeG+9KXQnwn84x/Axo3tXQERUfL4aVHXA/iOqp4CYDyAGSJyamLLipSTA3zhC6H5V15pz0cnIkquVoNaVXeq6vLg+WoAawAMTHRhjX35y6HzL7/c+vI7dgDvvstDpBJRxxdTH7WIFAIYA2BZlOumi0ipiJRWVFTEp7owV1wR6v5YvhzYvLn5ZXftAkaPBi6+GDjtNAvs9lRfD6xeDVRWtu/jElHnlOZ3QRHpAeBVALer6oHG16vqLACzAKC4uFjjVmFQ797AJZcA8+bZ/MsvA9/9bvRlf/QjwPutWL/eAvurXwUeeshGkSRSIABceWXo2CTHHQeMHAkMG2bXHToEfPYZkJ5uPz5XXx157G0iosZEtfVMFZF0APMA/FlVH25t+eLiYi0tLY1DeZF+/3vga1+z82PHAtEe4pNPgKFDgSNHml6XlWWHTz3nHODss+3ATzk5/h5bNTSeuyWPPgrcFsPm1r597TlNmwaccor/2xFR5yIiZapaHPW61oJaRATA7wDsVdXb/TxgooK6qgrIzwfq6mx+0yZg8ODIZaZNA556yqvDwu/ZZ5u/z7597T4GD7bukhtvjGx179sH/PSnwP/9HzBokLXKJ0yIfl8bN1pXy+HDNp+aCjQ0+H9+X/0q8OMfA4WF/m9DRJ3DsQb15wC8B+AjAIHgxfeqarMHHk1UUAPA5Ml2gCbAQi28+2PjRjs+iBeO774LXHghsHAhcOutwEcftX7/qanWdTF9OlBWBvzv/wL790cuc+21wM9+Bhx/fOiyQMACfNEimx81Cli2DCgvB1auBLZssS6OrCyb1q2zH5TG/16TkQHccQdwzz1Ar16xvDJEyRcIAJ9+Glqj9dZC6+psw/6RI7YNp2dPW5vNybFuQDrGoG6LRAb1734H/Pu/2/nG3R833BA6HOqECbbLuUfVwnLJEuDvf7dp9eq2jwrp0QP4z/8Err/e+p8fewy45Ra7LjXVQnrs2Jbvo6HBfkyeeAJ4/fXI6/Lzgf/6L+A//sMeixJP1UKkoSE0HTkSmmprbfuCN9XU2I9vdnboBzg11aaUFCAtzTaAZ2TYlJoaeX9ZWZF/OZcMn30G/OEP9l3IzAS6d7e60tMjX4eGBgvb+no7PXLEnr83bd8OfPyxNTyidTu2pGdPW1sdMsSmE06wxxexKS3NXqcePew0N9e6N5P92sVbpwrq5ro/Vq60bgfv6bz/vvVFtyQQsBEimzcDa9cCzz0H/O1vTZcbOhS4917gnXeA2bObXn/66cCGDfahB4D77gN+8IPYntfixdaSbvyy9e1rfd633OK/P52sy+qpp+x98YIyMxPo08e6tgoKgAED7L1fvBh47z37cfW6rdqDiHXNjRtn06hRtpY2YIAFZjTV1baBfPNmC7M+fWzq1cs++7W1oenw4dCPSn09kJdnG7cHDLA1vV//2j7zjdcYO4qCAluDHjLEnpP32vXubT+KaWl2mpNj73lGRrIrblmnCmoAuOyy0KiKG2+0X+M//jHUtTFpUqh7JFarVgFPPmkt8969ge99z1rwacHxMQsWADNmAGvWRL/9yJEWtm35UAQCtov8PffYFymciH0whwwBTjrJ+t+/8hWGd2NVVcAjjwA//7n9M1BH1bu3BXBmpk3dutlnoiMclCw311q7XrSoWv3elJpq782+ffZ+BQIt31+85Ofbd6h378gf7+OPD7Xmi4qs5Z6eHlobau5HM946XVA/8wwwdWrz1y9fDowZc2yP4b0s0UZ61NXZ6uKcOfaD4XWfpKYCS5daiB6L2lrr4vnJT1oeL56ZCXzxi8A3vgF8/vO2ut2SQADYts1WU4uKrPXRUR0+bBuJly611qLXbfHWW8fWQvS6LrzJ+7J6IZN1YBeyhxyHrCwgY+t61Jw4DIcO2bDLw9v3oKF3LgIB6yqorw91mdTW2mXe/aWnW1C1V0i15KSTgC9vfBCp9917tBVe96tfI/Wb00KvxQdLkH7e2UhPt0ZLxuL5yJh0ITIz7fnkz5+NonuvRWGhdWX4FQjY67B5s60db95sP0aBgE2q9n3zXuODB+36TZvs9W0PWVkW8vn5QL9969FjzLCj3TCZZe9Dzzk3VO+SpRjzrfFHR6fFotMF9b59tgoXrS9s+nQbodFeqquBN96w1ebJk23MdrzU19uPwUMPAf/8Z8tf6owMW7Xt189OvRa9iAXEli324Q7vkz/+eOtHLy62oB8/3v3Vw+pqW+N56CHbaNWS4cOBb37TgqWmxsJ9zx77oSovt71Xc3NtyKY3NRlnX1Jik0fE3pht2+zXbsMG61TNyLDrwr9PjW/baP7QvQ/gw0nfw7JldgybjxeXY2dKAXbtCnXtNZaebi2/oUMBWb8WVf1PRlWV/TilHtiLbv37Hv1hydq9BVmnFiIrC0hZswoV+SOwc6d199V/dgSXfzEN00f8HZ9/53tIWbzInkdenk1//rNthMnPt/mbbrJVxQED7LL09MjnGuNzj5hvbdko6uos1NetAz558k3sGD0JO3fae3pozSdoGHgi6uvtrarYdAA7D/Vqtx/FKVOid5G2ptMFNWBdE7/8pfXhjhplXQ6nn27n/Yx37miOHLGw3bjRNvzMnm1rDvGUlQWcdx7wb/9mq93eFz47277DJ55o39nGLXdVy62yMqupujpyVbdvX/tRGDjQvuf799tz2brVAjMlxR4jO9ses7LSwmTnTgtWbwy7CPCvf9kPdUuGDgVmzrQvTOoPSvwHRrR5EdsqvWSJpekf/2hPKryVIGJPbMcOGxJUWGjTN79pG0+OP95e0JSUpuHW0GAbSJYssVWjp59GoFcf7E3LR/VVN6Dm/eWoyeiNmiMp6HfOUBTVbTjaDdckHFua984fPAjs2AEdPhySk9P6ixmNd1+nnQb072+tpmefBe680/oJune3jTqvvWbX9e9vr0dNjb123pvZuLbmnoef96iV16G+TrFrl33eDp19IWr/NP/oxuGtN9yLTVfdic1ra7F1ZwZq9tegrkcO6gKpOPyZ4ghia7185YrDmPNa7P0lnTKoCVixAvjNb4AXX/S/u3p+vuXG+vWhjZ+xyMiw+wjvDigvT87u8gMHAt/+tmVkSopN+fk24ictDRZ+55xj/SEnnmi/Nr16WQept5Vt8GD7hampsfmLL7aNBBs2WHPNz5hOP7p3tyb9mWdaMz4311obfsIyLc3Cbvt2a057Nm2y8ad9+9r9zJoFfOtbdl1Dg81feql1BC9ZYn0S1dWR9z1okG3Fvv12G7axe7ft1jt5MnDRRfE//kJqqv0iHzhgras+fWxr7oQJodWeFSuACy4I9Tc8+qgNq/LG802aBDz4oG0oWrPGWvpFRfYBELHWzIUX2uplbq4Nq7r4Ynv9du0C9u61zuqsLHttV69utlwFUI2e2I187EY+KtEPh5CNQ8jGQfRALTKQggBSEIBAkYIAhmE9JuEtay20smYQjkHdBRw6ZN+vigoLzbo6a1R4rdGBA62l2bu3Le815MrKbKji/Pkd5/CxgwcDd99tG5IzflTStLU1Y4attj/3XHwf+Oqr7Y87q6tti5PXcquvtxAoLASuuspa3R3JzJnA/ff7b53X11vXx/Ll1v+0cyfw9a9bOM6f3761x9vnPx996BdgYf+XvwBPP21foEDA1poef9xem0DAdtjYu7dNW/kZ1J1Ba/12sa7SR+kj3LLFGlArn1qK2jHjj24I27/fujY++aT5xl+vXvZHxGPH2o+Ct4NDba39cGzfHmrQ9OplDblBg6yRKxI6BkpNjTUQjzvOprw8a4QB9l3IzrYNxWlpsFZZ7962gcDroO/Vy7ay1tT4fmmb9Y1v2Bi/1roTol3XeN6r9e9/t/6cPXtsONHWraEXobnb19RYIBYWWkvfezGGDbO+5L177Y25+WZreXr3dfPNdnCcPn2sA37vXjsf7bHi0L3ga97bMpiTYxteqqosHN9919Y6MjPtQ3TDDf5+aKdOBX77W2tleGE5fLiNpa2stOnWW22rv9f3lp9vr/tnn9kHdPTo+LzH0eZj0FJQQ1XjPo0dO1YpzoDI+ZkzQ+e3bbPrA4HI5Q8dUl28WPWJJ2x+6VLVTz+15cLvzztfX6+6aZPN798f9bGqq1U/vu3nun696sqVqsuXq2645Rfa0NBMbfGYj2blStVBg7yVhqbTRRepbtgQ+TwbP2/V2OfDa2ut7mN9rHjOt7Zsa2J9z+JZW3vPhz+Xlq7zMx8DAKXaTKYyqBOppS+139vW1anOnm1v1a9/rfraa6pLltj8TTepFhWFwqlHD9XRo1WvvtrmU1Ojh1hWlp0WF6tOnGjnR41SzciIXG7oUNUpU+z8//xPaAJU581T3bjRwt37kThwQHXr1ug/GqqqR46orl9v85WV9tzCr2+8fLTXbeZM1RtvbD6gw6eZM+P7pY1VvH+wjmU+joHiSzxri/U9imfDINGvUxgGdbK01Grw8+G84gp/gZTMyQv3tLTIy3NyVM86S/X6621++PCmywCqPXva6VlnqV5+uerXv27zDz6o+uSTdv6991Q3b1Y9fDjysb70pcR/iSn5ush71FJQs486Ed54w45dum+fjXU78UTbwv/ww6FxqhMnWr/Z3r023Xqr7Qbp7fv7+OOh+xs61PomTz/d+vUa+/73bZ/1PXtsuY8/tmFiBw+GDogQ3ne2f7/1VS5dan14kyfb0LPhw21kgIjtB//DHyb+tWqrO++0PYJSU0PPCzimPkKiZOLGxPZ01112XNR48oK4pUA6lvljua23kaymJnIvm127bAzghg22s8SKFfaDk5UVun0gYLfPybGDs3hDw6ZP9/e6zJxppzHuLEHkoq67MTEZq0zeqv7559vp/Pmqzzxj58eN89ed4PUb19c3X3s8V/Fd3igWfr6hoemyRJ0EOm0fdWvBm8gNRdG88YY9RvfutqGtpT7qWOcba89+umRuBIv1dSHqoDpnUHtDrSLGhYV5+WW7vrhYdeRI1ZNOsvmbb7brdu8+9jAMv76qKrRh7OGHm15/rFuyu6ousiGJqHMG9UMPWfmXXaZ68GDocm9Ilt9p7tzQULJYuxfCr582zebHj4/ssmgOA4iIwrQU1B1vY2JJie3u2tidd9oIh5NOsn2Lvef1wQehg/oOG2bHFFiwoOntp0yxg2a8/rpt2Fq61HYlLSmxf8I96ywbKfH443b7xYttg5n3VxjeoblWrQJOPTUxz52IOq3ON+rjxRdt+Blgu9Vu2WIHq9m0yQ7MEgjYKInvf7/50QqHD9sIhAEDEnM09hgPyEJEXVtLQd3KoeYd9dhjofNLlthBJjZtsvlAwALyvvtCw7c84fPe3zY09w8E3g/B+PHRr//2t+3UOyK89wPgdaowpIkoTvz8C/nTACYD2K2qI/3caUJb1B9+aMHcu7cddKZbt+hdIX5atOFjblWjHy/Ym29osCMBNXd9tHkiIp9aalGnRbuwkWcAPAbg9/Esqs1++Us7nTrVjkkLWNhGC9rWND46WGPhLXDvEG7NXR9tnogoDlrt+lDVRQD2tkMtraustH9/FbFjDoeLx9+6NA7axi3y1q5ndwcRJUDc+qhFZLqIlIpIaUVFRbzuNtJTT1mf8KWX2uiOxo61Reunq4SIqJ3FLahVdZaqFqtqcV5eXrzuNqS+3v5SBwBuuSX6MgxSIuqEOs6oj7fftr8ZAYBLLkluLURE7ahjBHVJCXD55aH51FTrk2YLmoi6gFZHfYjIbADnA+gnIuUAZqrqbxJdWISSEvsn1jlzbJ5D4IioC2k1qFX12vYopFXr1ye7AiKipOgYXR+qoaC+667k1kJE1M46RlDv2GF/Md+vn/39EhFRF9IxgtprTQ8bltw6iIiSoGME9bp1djp8eHLrICJKgo4R1GxRE1EX1rGCmi1qIuqCOkZQe10fbFETURfkflAfOQJ8/LHtiThkSLKrISJqd+4H9ebNdtD+wkL730Mioi7G/aDmhkQi6uI6TlBzQyIRdVHuBzU3JBJRF+d+ULNFTURdXMcJaraoiaiLcjuoDxwAdu0CuncHCgqSXQ0RUVK4HdRea3roUCDF7VKJiBLF7fTjhkQiIseDmhsSiYg6SFCzRU1EXZjbQc2uDyIif0EtIhNFZJ2IbBSRuxNaUUmJnYb/TyKDmoi6sFaDWkRSATwO4FIApwK4VkROjXsldXVARQVw//12umqV/U8iAPTtG/eHIyLqKPy0qMcB2Kiqm1X1CIAXAVwZ90pWrADy8+18fj4walTcH4KIqCPyE9QDAWwLmy8PXhZBRKaLSKmIlFZUVMRWRUkJMG5c89eL2OR1ixARdSF+glqiXKZNLlCdparFqlqcl5cXWxUlJdYnrerdWfR5BjURdUF+grocwAlh8wUAdiSmHCIiasxPUP8DwFARKRKRbgCmAHg9YRXNnNnyPBFRFyOqTXoxmi4kMgnAIwBSATytqg+0tHxxcbGWlpbGp0Iioi5ARMpUtTjadWl+7kBV3wTwZlyrIiIiX9zeM5GIiBjURESuY1ATETmOQU1E5Dhfoz5ivlORCgBb23jzfgAq41hOPLG2tmFtbcPa2qaj1jZIVaPuLZiQoD4WIlLa3BCVZGNtbcPa2oa1tU1nrI1dH0REjmNQExE5zsWgnpXsAlrA2tqGtbUNa2ubTlebc33UREQUycUWNRERhWFQExE5zpmgbtc/0PVXz9MisltEVoZd1ldE/iIiG4KnOUmo6wQRWSAia0RklYjc5lBtmSLygYj8M1jb/cHLi0RkWbC2OcHD5SaFiKSKyIciMs+l2kRki4h8JCIrRKQ0eFnS39NgHX1E5BURWRv83J3tQm0iMjz4ennTARG53YXagvXdEfwerBSR2cHvR5s+b04Edbv9gW5sngEwsdFldwOYr6pDAcwPzre3egDfUdVTAIwHMCP4WrlQWy2AC1T1dACjAUwUkfEAfgLg58Ha9gG4KQm1eW4DsCZs3qXaJqjq6LBxti68pwDwCwBvq+rJAE6HvSZ4O2EAAAMcSURBVH5Jr01V1wVfr9EAxgL4DMBcF2oTkYEAbgVQrKojYYeInoK2ft5UNekTgLMB/Dls/h4A9zhQVyGAlWHz6wAMCJ4fAGCdAzW+BuBi12oDkAVgOYCzYHtipUV7r9u5pgLYF/cCAPNgfzPnSm1bAPRrdFnS31MAvQB8jODAA5dqa1TPJQDed6U2hP5rti/scNLzAHyhrZ83J1rU8PkHug7or6o7ASB4mp/MYkSkEMAYAMvgSG3BroUVAHYD+AuATQCqVLU+uEgy39tHANwFIBCcz4U7tSmAd0SkTESmBy9z4T0dDKACwG+DXUZPiUi2I7WFmwJgdvB80mtT1e0AfgbgEwA7AewHUIY2ft5cCWpff6BLISLSA8CrAG5X1QPJrsejqg1qq6IFAMYBOCXaYu1bFSAikwHsVtWy8IujLJqsz925qnoGrPtvhoicl6Q6GksDcAaAX6nqGACHkLwumKiC/bxXAHg52bV4gv3iVwIoAnA8gGzYe9uYr8+bK0HdUf5A91MRGQAAwdPdyShCRNJhIf28qv7Bpdo8qloFYCGsH72PiHj/JpSs9/ZcAFeIyBYAL8K6Px5xpDao6o7g6W5YP+s4uPGelgMoV9VlwflXYMHtQm2eSwEsV9VPg/Mu1HYRgI9VtUJV6wD8AcA5aOPnzZWgbt8/0G271wF8LXj+a7D+4XYlIgLgNwDWqOrDjtWWJyJ9gue7wz6sawAsAHBNMmtT1XtUtUBVC2Gfr7+q6vUu1CYi2SLS0zsP629dCQfeU1XdBWCbiAwPXnQhgNUu1BbmWoS6PQA3avsEwHgRyQp+Z73XrW2ft2RuAGjU+T4JwHpYn+b3HKhnNqxvqQ7WqrgJ1qc5H8CG4GnfJNT1Odjq0r8ArAhOkxyp7TQAHwZrWwngv4OXDwbwAYCNsNXTjCS/t+cDmOdKbcEa/hmcVnmffxfe02AdowGUBt/XPwLIcai2LAB7APQOu8yV2u4HsDb4XXgWQEZbP2/chZyIyHGudH0QEVEzGNRERI5jUBMROY5BTUTkOAY1EZHjGNRERI5jUBMROe7/AZbm7cxyM5ucAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lr = LinearRegression()\n",
    "plot_learning_curves(lr, X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "现在让我们看一个在相同数据上 10 阶多项式模型拟合的学习曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD4CAYAAAAZ1BptAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3RV5Z3/8feXEEC8cbdIrKEtywLWokZEnc54GTXSVlitnYV1jawOlU6htnZNO+KvvxaodcYuZ2rLGrVDlYodq7VoK78WtdRr7aAYxCoXLfFKBCVcS70Age/vj/2EbA7nhOSc5Ownyee11lnZ+9nP3uebnJAv3+fZzznm7oiIiOTTK+sAREQkXkoSIiJSkJKEiIgUpCQhIiIFKUmIiEhBvbMOoKMNGTLEq6ursw5DRKRLWbFixWZ3H5rb3u2SRHV1NXV1dVmHISLSpZjZ6/naNdwkIiIFKUmIiEhBShIiIlJQt5uTEBFprz179tDQ0MD777+fdSidrl+/flRVVVFZWdmm/koSItLjNTQ0cOSRR1JdXY2ZZR1Op3F3tmzZQkNDAyNHjmzTORpuEpEe7/3332fw4MHdOkEAmBmDBw9uV8WkJNEFvPMOvPZa1lGIdG/dPUE0a+/3qSQRuR07oLoaRo6E//mfrKMRkZ5GSSJyjz0Gmzcn24sWZRqKiHSi7du3c/PNN7f7vIkTJ7J9+/ZOiCihJBG59NDhrl3ZxSEiecyZ02GXKpQk9u7d2+p5S5YsYcCAAR0WRy4licjt2dOyvXt3dnGISB5z53bYpWbNmsXLL7/MuHHjOO200zjnnHP4/Oc/z8c+9jEAJk+ezKmnnsrYsWOZP3/+/vOqq6vZvHkzr732GqNHj+aKK65g7NixXHDBBbz33nslx6UkEbl0YkgnDBHpJGZtf7Sn/yFcf/31fPjDH+a5557jhhtuYPny5Vx33XWsWbMGgAULFrBixQrq6uqYN28eW7ZsOega69atY+bMmaxevZoBAwZw7733lvzj0DqJyKmSEOmZxo8ff8Bahnnz5vGrX/0KgPXr17Nu3ToGDx58wDkjR45k3LhxAJx66qm81gG3RSpJRE6VhEiZube9r1n7+rfD4Ycfvn/7scce4/e//z3Lli2jf//+nH322XnXOvTt23f/dkVFhYabegJVEiI9w5FHHsnOnTvzHtuxYwcDBw6kf//+vPjiizz11FNli0uVROTSiUFJQiQys2d32KUGDx7MWWedxYknnshhhx3GMcccs/9YbW0tP/7xjznppJM44YQTmDBhQoc976EoSUQuXUlouEkkMh14CyzAz3/+87ztffv25YEHHsh7rHneYciQIaxatWp/+ze+8Y0OiUnDTZFTJSEiWTpkkjCzBWa2ycxWpdpuMLMXzex5M/uVmQ1IHbvGzOrN7CUzuzDVXhva6s1sVqp9pJk9bWbrzOwXZtYntPcN+/XheHVHfdNdiSoJEclSWyqJ24HanLalwInufhLwZ+AaADMbA0wBxoZzbjazCjOrAG4CLgLGAJeGvgDfB25091HANmBaaJ8GbHP3jwA3hn49jioJEcnSIZOEuz8BbM1p+527N4Xdp4CqsD0JuNvdd7n7q0A9MD486t39FXffDdwNTLLk7QjPBZrflWghMDl1rYVhexFwnvWUt2lM0d1NIpKljpiT+CegeUZlBLA+dawhtBVqHwxsTyWc5vYDrhWO7wj9D2Jm082szszqGhsbS/6GYqJ1EiKSpZKShJl9C2gC7mxuytPNi2hv7VoHN7rPd/cad68ZOnRo60F3MblzEp20bkdEJK+ik4SZTQU+BVzmvv9PVwNwXKpbFbChlfbNwAAz653TfsC1wvGjyRn26glyh5iamvL3E5Ge5YgjjijL8xSVJMysFrgauNjd300dWgxMCXcmjQRGAcuBZ4BR4U6mPiST24tDcnkUuCScPxW4P3WtqWH7EuCRVDLqMXKHmDQvISLldMjFdGZ2F3A2MMTMGoDZJHcz9QWWhrnkp9z9n919tZndA6whGYaa6e57w3W+AjwEVAAL3H11eIqrgbvN7HvASuC20H4b8DMzqyepIKZ0wPfb5eQmhd27IfWWLiLSTVx99dUcf/zxzJgxA4A5c+ZgZjzxxBNs27aNPXv28L3vfY9JkyaVNS7rbv85r6mp8bq6uqzD6DCf+hT89rct+2+/DcOGZRePSHe0du1aRo8eDbTpXb2L1tqf25UrV3LVVVfx+OOPAzBmzBgefPBBBgwYwFFHHcXmzZuZMGEC69atw8w44ogj+Otf/1pUHOnvt5mZrXD3mty+eluOyOWrJESk+zn55JPZtGkTGzZsoLGxkYEDBzJ8+HC+/vWv88QTT9CrVy/efPNN3n77bT7wgQ+ULS4licjlzknoNliR7uuSSy5h0aJFvPXWW0yZMoU777yTxsZGVqxYQWVlJdXV1XnfIrwzKUlETpWESHllOQI/ZcoUrrjiCjZv3szjjz/OPffcw7Bhw6isrOTRRx/l9ddfL3tMShKR091NIj3H2LFj2blzJyNGjGD48OFcdtllfPrTn6ampoZx48bx0Y9+tOwxKUlELjcpaLhJpHt74YUX9m8PGTKEZcuW5e1X7KR1e+mtwiOnSkJEsqQkETlVEiKSJSWJyKmSECmP7rZmrJD2fp9KEpHT3U0ina9fv35s2bKl2ycKd2fLli3069evzedo4jpyWich0vmqqqpoaGigu33UQD79+vWjqqrq0B0DJYnIqZIQ6XyVlZWMHDky6zCipOGmyKmSEJEsKUlETpWEiGRJSSJi+/bB3r0HtilJiEg5KUlELN/QkoabRKSclCQili8hqJIQkXJSkohYvoSgSkJEyklJImKqJEQka0oSEcuXEJQkRKSclCQipuEmEcmakkTENNwkIllTkoiYKgkRydohk4SZLTCzTWa2KtU2yMyWmtm68HVgaDczm2dm9Wb2vJmdkjpnaui/zsymptpPNbMXwjnzzMxae46eRJWEiGStLZXE7UBtTtss4GF3HwU8HPYBLgJGhcd04BZI/uADs4HTgfHA7NQf/VtC3+bzag/xHD2GJq5FJGuHTBLu/gSwNad5ErAwbC8EJqfa7/DEU8AAMxsOXAgsdfet7r4NWArUhmNHufsyT97I/Y6ca+V7jh5DK65FJGvFzkkc4+4bAcLXYaF9BLA+1a8htLXW3pCnvbXnOIiZTTezOjOr607vB69KQkSy1tET15anzYtobxd3n+/uNe5eM3To0PaeHi1VEiKStWKTxNthqIjwdVNobwCOS/WrAjYcor0qT3trz9FjqJIQkawVmyQWA813KE0F7k+1Xx7ucpoA7AhDRQ8BF5jZwDBhfQHwUDi208wmhLuaLs+5Vr7n6DF0d5OIZO2QH19qZncBZwNDzKyB5C6l64F7zGwa8AbwudB9CTARqAfeBb4A4O5bzexa4JnQ77vu3jwZ/mWSO6gOAx4ID1p5jh5D6yREJGuHTBLufmmBQ+fl6evAzALXWQAsyNNeB5yYp31LvufoSVRJiEjWtOI6YqokRCRrShIRUyUhIllTkoiY7m4SkawpSURM6yREJGtKEhFTJSEiWVOSiJgqCRHJmpJExFRJiEjWlCQiprubRCRrShIR0zoJEcmakkTEVEmISNaUJCKWLyHs3Qv79pU/FhHpmZQkIlZoaElDTiJSLkoSESs0tKQhJxEpFyWJiKmSEJGsKUlETJWEiGRNSSJiqiREJGtKEhFTJSEiWVOSiFihikFJQkTKRUkiYoWSgYabRKRclCQipkpCRLKmJBGxdDI4/PCWbVUSIlIuShIRSyeDdJJQJSEi5VJSkjCzr5vZajNbZWZ3mVk/MxtpZk+b2Toz+4WZ9Ql9+4b9+nC8OnWda0L7S2Z2Yaq9NrTVm9msUmLtigpVEkoSIlIuRScJMxsBfBWocfcTgQpgCvB94EZ3HwVsA6aFU6YB29z9I8CNoR9mNiacNxaoBW42swozqwBuAi4CxgCXhr49RqFKQsNNIlIupQ439QYOM7PeQH9gI3AusCgcXwhMDtuTwj7h+HlmZqH9bnff5e6vAvXA+PCod/dX3H03cHfo22OokhCRrBWdJNz9TeA/gDdIksMOYAWw3d2bQrcGYETYHgGsD+c2hf6D0+055xRqP4iZTTezOjOra2xsLPZbio4qCRHJWinDTQNJ/mc/EjgWOJxkaCiXN59S4Fh72w9udJ/v7jXuXjN06NBDhd5lqJIQkayVMtz098Cr7t7o7nuA+4AzgQFh+AmgCtgQthuA4wDC8aOBren2nHMKtfcYurtJRLJWSpJ4A5hgZv3D3MJ5wBrgUeCS0GcqcH/YXhz2CccfcXcP7VPC3U8jgVHAcuAZYFS4W6oPyeT24hLi7VLctU5CRLLX+9Bd8nP3p81sEfAs0ASsBOYDvwXuNrPvhbbbwim3AT8zs3qSCmJKuM5qM7uHJME0ATPdfS+AmX0FeIjkzqkF7r662Hi7mr17W7YrKqBfv5Z9VRIiUi5FJwkAd58NzM5pfoXkzqTcvu8DnytwneuA6/K0LwGWlBJjV5VOBJWVyaOZKgkRKRetuI5UOhH06ZM8mqmSEJFyUZKIVGuVhJKEiJSLkkSkWqskNNwkIuWiJBGp3EpCw00ikgUliUjlVhKauBaRLChJREqVhIjEQEkiUq1VEkoSIlIuShKRaq2S0HCTiJSLkkSktE5CRGKgJBEprbgWkRgoSURKlYSIxEBJIlJacS0iMVCSiJRWXItIDJQkIqV1EiISAyWJSGmdhIjEQEkiUlonISIxUJKIlCoJEYmBkkSkVEmISAyUJCKlSkJEYqAkESnd3SQiMVCSiJTWSYhIDJQkIqUV1yISg5KShJkNMLNFZvaima01szPMbJCZLTWzdeHrwNDXzGyemdWb2fNmdkrqOlND/3VmNjXVfqqZvRDOmWdmVkq8XYkqCRGJQamVxI+AB939o8DHgbXALOBhdx8FPBz2AS4CRoXHdOAWADMbBMwGTgfGA7ObE0voMz11Xm2J8XYZqiREJAZFJwkzOwr4W+A2AHff7e7bgUnAwtBtITA5bE8C7vDEU8AAMxsOXAgsdfet7r4NWArUhmNHufsyd3fgjtS1uj29C6yIxKCUSuJDQCPwUzNbaWa3mtnhwDHuvhEgfB0W+o8A1qfObwhtrbU35Gk/iJlNN7M6M6trbGws4VuKR24lUVEBvcKr5Q5792YTl4j0LKUkid7AKcAt7n4y8A4tQ0v55JtP8CLaD250n+/uNe5eM3To0Naj7iJyKwnQkJOIlF8pSaIBaHD3p8P+IpKk8XYYKiJ83ZTqf1zq/CpgwyHaq/K09wi5lQRo8lpEyq/oJOHubwHrzeyE0HQesAZYDDTfoTQVuD9sLwYuD3c5TQB2hOGoh4ALzGxgmLC+AHgoHNtpZhPCXU2Xp67V7amSEJEY9C7x/CuBO82sD/AK8AWSxHOPmU0D3gA+F/ouASYC9cC7oS/uvtXMrgWeCf2+6+5bw/aXgduBw4AHwqNHOFQloSQhIuVQUpJw9+eAmjyHzsvT14GZBa6zAFiQp70OOLGUGLuqfJWEhptEpNy04jpS+SoJDTeJSLkpSURKlYSIxEBJIlKqJEQkBkoSkTpUJaEkISLloCQRKa2TEJEYKElESuskRCQGShKRUiUhIjFQkoiUKgkRiYGSRKS04lpEYqAkESmtkxCRGChJRErrJEQkBkoSkVIlISIxUJKIlCoJEYmBkkSE9u078ONJe4f36tXEtYiUm5JEhNJDSZWVYOGDXDXcJCLlpiQRoXzzEaDhJhEpPyWJCOWbjwBVEiJSfkoSEVIlISKxUJKIUDoBpJOEJq5FpNyUJCKUO3HdTMNNIlJuShIRKlRJaLhJRMqt5CRhZhVmttLMfhP2R5rZ02a2zsx+YWZ9QnvfsF8fjlenrnFNaH/JzC5MtdeGtnozm1VqrF2FKgkRiUVHVBJfA9am9r8P3Ojuo4BtwLTQPg3Y5u4fAW4M/TCzMcAUYCxQC9wcEk8FcBNwETAGuDT07fZUSYhILEpKEmZWBXwSuDXsG3AusCh0WQhMDtuTwj7h+Hmh/yTgbnff5e6vAvXA+PCod/dX3H03cHfo2+21pZJQkhCRcii1kvgh8K/AvrA/GNju7k1hvwEYEbZHAOsBwvEdof/+9pxzCrUfxMymm1mdmdU1NjaW+C1lry2VhIabRKQcik4SZvYpYJO7r0g35+nqhzjW3vaDG93nu3uNu9cMHTq0lai7BlUSIhKL3iWcexZwsZlNBPoBR5FUFgPMrHeoFqqADaF/A3Ac0GBmvYGjga2p9mbpcwq1d2ttWSehSkJEyqHoSsLdr3H3KnevJpl4fsTdLwMeBS4J3aYC94ftxWGfcPwRd/fQPiXc/TQSGAUsB54BRoW7pfqE51hcbLxdSaFKQhPXIlJupVQShVwN3G1m3wNWAreF9tuAn5lZPUkFMQXA3Veb2T3AGqAJmOnuewHM7CvAQ0AFsMDdV3dCvNHRimsRiUWHJAl3fwx4LGy/QnJnUm6f94HPFTj/OuC6PO1LgCUdEWNX0pZKQsNNIlIOWnEdIVUSIhILJYkIacW1iMRCSSJCWnEtIrFQkoiQ1kmISCyUJCKkFdciEgsliQipkhCRWChJREgrrkUkFkoSEdKKaxGJhZJEhLROQkRioSQRobauuPa874krItJxlCQiVKiS6NULKipa9puaEBHpVEoSESpUSYAmr0WkvJQkIlSokgBNXotIeSlJZOQvfyl8LP3Hv7VKQklCRDqbkkQGvvQlOPpouPLK/MfTw0itVRIabhKRzqYkUWbvvgs/+UmyfdNNsG3bwX1USYhILJQkymzDhpZbV91h2bKD+7RWSWjiWkTKSUmizDZuPHD/yScP7tNaJaGJaxEpJyWJMtuw4cD9fEmirZWEkoSIdDYliTLLTRLLl8OuXQe2tbWS0HCTiHQ2JYkyyx1u2rULVqw4sE2VhIjEQkmizHIrCTh4yKmtdzepkhCRzlZ0kjCz48zsUTNba2arzexroX2QmS01s3Xh68DQbmY2z8zqzex5Mzslda2pof86M5uaaj/VzF4I58wzMyvlm41BbiUBByeJtq6TUCUhIp2tlEqiCfgXdx8NTABmmtkYYBbwsLuPAh4O+wAXAaPCYzpwCyRJBZgNnA6MB2Y3J5bQZ3rqvNoS4o1Cvkrij3+Effta9rVOQkRiUXSScPeN7v5s2N4JrAVGAJOAhaHbQmBy2J4E3OGJp4ABZjYcuBBY6u5b3X0bsBSoDceOcvdl7u7AHalrdVnpJNG/f/J161Z48cWWdq24FpFYdMichJlVAycDTwPHuPtGSBIJMCx0GwGsT53WENpaa2/I057v+aebWZ2Z1TU2Npb67XSad95pec+mPn3g/PNbjqWHnFRJiEgsSk4SZnYEcC9wlbu38rZ15JtP8CLaD250n+/uNe5eM3To0EOFnJn0fMTw4fCJT7Tsp5OEVlyLSCxKShJmVkmSIO509/tC89thqIjwdVNobwCOS51eBWw4RHtVnvYuK50kjj22cJLQimsRiUUpdzcZcBuw1t1/kDq0GGi+Q2kqcH+q/fJwl9MEYEcYjnoIuMDMBoYJ6wuAh8KxnWY2ITzX5alrdUnp+Yjhw+Hkk+Gww5L9V1+FN99MtrVOQkRiUUolcRbwj8C5ZvZceEwErgfON7N1wPlhH2AJ8ApQD/wEmAHg7luBa4FnwuO7oQ3gy8Ct4ZyXgQdKiDdz6SRx7LFJVTBhQkvbk08mb/qnFdciEovexZ7o7k+Sf94A4Lw8/R2YWeBaC4AFedrrgBOLjTE2uXMSAH/zN/Doo8n2k0/CZz/b0qeiIvlc6zRVEiJSTlpxXUa5lQQkSaLZ0qUHvo9TbhWR26ZKQkQ6m5JEGeVOXAOceWbLeomXXoJFi1r65M5H5LapkhCRzqYkUUa5E9cARxwBM2a0tM+Z07Kdr5JQkhCRclKSKKN8w00A3/xmy11Or73W0p6vktBwk4iUk5JEmeSuth40qOXYsGEwM8+UvioJEcmakkSZ5N7ZlPt+tt/4Rks10SxfJdGvX8v2pk0HHxcR6UhKEmWS7/bXtGOOOXBuAvJXEjU1LdsPPqhqQkQ6l5JEmRSaj0hLz01A/kri1FPh+OOT7e3b4ZFHOi5GEZFcShJl0pYkkVtN5EsSZvCZz7Ts33tvx8QnIpKPkkSZHGq4qdk3vwkjwhuiX3hh/j7pVdm//jU0NZUen4hIPkW/LYe0T1sqCUiqiZUrkzf8O+20/H3OOCNJNBs3wubN8Ic/wDnndGy8IiKgSqJs8i2kK2ToUBg//uA7oJr16qUhJxEpDyWJMsn3lhylSA853XffgZ+RLSLSUZQkyqQ9lURbfOITMGRIsr1xIzz1VOnXFBHJpSRRBunV1pWVMHhw6dfs3RsmT27Z15CTiHQGJYkyyB1qKjTX0F7pIad7700+sEhEpCMpSZRBRw81NTv3XBgwINl+/XWoq+u4a4uIgJJEWXT0pHWzPn3g4otb9r/4xZZhLRGRjqAkUQadVUkAXHllMj8B8PzzcMklegtxEek4ShJF2rQJ1q9v22rnzqokIHnDv1tvbdlfuhS+9CXNT4hIx1CSKMIPfpD8sf/gB5Mhn2OPTVZHT5sGd955YFKAzq0kAKZOhblzW/Z/+lP47neVKESkdHpbjnZwh6uvhhtuOLBt48bkUVcHCxYk7WPGJIlj9Gj4059a+nd0JdHs299O3srj9tuT/Tlz4IEHYPZsqK3tuDuqRKRnib6SMLNaM3vJzOrNbFanPln6A6Zz9pua4J9OWXlAgjiiz66Cf3zXrIGFC2HWLFi1qqV9f5Jo5bmK2be5c5g/H84/v6Xt6adh4kSYMAFu/uRvuf9+eOYZaGiAXf/32gPOX3flPK6/Phm+6tsXThm+kWuvhRdeAJ/dsbF26H5MsSi27h9b7n5MseTb7wBRVxJmVgHcBJwPNADPmNlid1/Tkc+zYgXc+l/vw+3HwGvvtRxYeAz7XnkPB9b+uTdPPnfy/kMX1+7i7gcH0PvNN9j4di9efaOCJyb/Jw+f+W2W1VWye/fB2aMXeznusG3Q6Mn4UPozS0vZ9+R6lTNmcN9/G//nusP579sq2E1fAJYvh+V8Epako/k2R/5oH0MHO716OfWvfvWAWFe+NZyV34HvfAeqmEb/O5vYvdvYtRv2vv1l+vxkL30qoU8fp3LdZ+i1aA8VFVDRC+y5i7D/1zJ7bs9eSO8Hd1NRAb0roNcfzoTHdmPmmIE9ejr8oSXh7nz4PLbe3cTWbcb2v/SC96+m8j/30bs3VFQAW2bCf+3DHXptncGQu5oYNmQfw4bsY9DiD9Dr9XdbvpHbh0FW+1k+t2LrsbH+6Fv5P7CsWOYRD1yb2RnAHHe/MOxfA+Du/17onJqaGq9r54KBX/4S/uEf2t7/CyxgPtPpzd68x9/lMJYzntWMZS2jWcMY3mQEl3MH3+Lf2hVbsd7kWL7P1cxnOrvod+gTRKRbeO+9Az/muK3MbIW71+S2R11JACOA9an9BuD03E5mNh2YDvDBD36wfc8wZw7MXQ38sk3dr+Z6/p1raG2Ivz/vcTaPczaPty+WDjSCDczja8ziehYylZf5MBs4lo0MZyPD2cwQ9qZe/r68Ty0P8lnu5e94nP/lTH7NZJYwkZ0cldn3ISLtdFg/YFcyIdkBw0+xVxKfAy509y+G/X8Exrv7lYXOKaaSePll+N3vgBlfhptv2d/uM2ZgN9+cDIkYnPbPp3CKP5sO8MBbiNqzX8q5HbC/z3qxY+s+GhuTBXgnnHYkR/rOg/rv2gUv9xtDxYtr6NMnuZurouoD7HnjLXbvTtZk7B59EnuffZ59+2DvXth3+gRY1vKOg/vOOJO9T/wvTU3J3I5fcAH+4O9wDyFNvAj/7QP794/89N8x6PnHGTQoWVFuR/Snace7+8/nmGHYpk0ANA0bzubnN7JpU3Jb8rbPz4Cbbm75PmZmuJ/lcyu2Hhvr9D0371871R6FKonYk0RZhptST1i+P9QZJ4luE2tMsSi27h9bV4u1HQoliYo5nTAb3lHmzp37FjBn7ty5i+fOnfsuMA/4tzlz5jQWOmf+/Plzpk+fXvyTnn12+fbL+Vyl7scUi2JTbDHtxxRLvv02mjt37sY5c+bMz22PupIAMLOJwA+BCmCBu1/XWv+SKgkRkR6qq05c4+5LyLl5U0REyiP6xXQiIpIdJQkRESlISUJERApSkhARkYKiv7upvcysEXi9yNOHAJs7MJyOpNiKo9iKo9iK05VjO97dh+Y2drskUQozq8t3C1gMFFtxFFtxFFtxumNsGm4SEZGClCRERKQgJYkDHbQkPSKKrTiKrTiKrTjdLjbNSYiISEGqJEREpCAlCRERKUhJIjCzWjN7yczqzWxWxrEsMLNNZrYq1TbIzJaa2brwdWBGsR1nZo+a2VozW21mX4slPjPrZ2bLzexPIba5oX2kmT0dYvuFmfUpd2whjgozW2lmv4kprhDLa2b2gpk9Z2Z1oS3z1zTEMcDMFpnZi+H37owYYjOzE8LPq/nxFzO7KobYQnxfD/8OVpnZXeHfR7t/55QkSP7xAjcBFwFjgEvNbEyGId0O1Oa0zQIedvdRwMNhPwtNwL+4+2hgAjAz/KxiiG8XcK67fxwYB9Sa2QTg+8CNIbZtwLQMYgP4GrA2tR9LXM3OcfdxqXvpY3hNAX4EPOjuHwU+TvIzzDw2d38p/LzGAacC7wK/iiE2MxsBfBWocfcTST5qYQrF/M65e49/AGcAD6X2rwGuyTimamBVav8lYHjYHg68lPXPLcRyP3B+bPEB/YFnST4TfTPQO99rXcZ4qkj+YJwL/AawGOJKxfcaMCSnLfPXFDgKeJVwk01MseXEcwHwx1hiA0YA64FBJB8J8RvgwmJ+51RJJJp/oM0aQltMjnH3jQDh67CM48HMqoGTgaeJJL4wpPMcsAlYCrwMbHf3ptAlq9f2h8C/AvvC/uBI4mrmwO/MbIWZNX+0Ywyv6YeARuCnYajuVjM7PJLY0qYAd4XtzGNz9zeB/wDeADYCO4AVFPE7pySRsDxtuje4FWZ2BHAvcJW7/yXreP8jhB8AAAIISURBVJq5+15Pyv8qYDwwOl+3csZkZp8CNrn7inRznq5Z/s6d5e6nkAy5zjSzv80wlrTewCnALe5+MvAO2Q175RXG9S8Gfpl1LM3CPMgkYCRwLHA4yWub65C/c0oSiQbguNR+FbAho1gKedvMhgOEr5uyCsTMKkkSxJ3ufl9s8QG4+3bgMZJ5kwFm1vwpjFm8tmcBF5vZa8DdJENOP4wgrv3cfUP4uolkXH08cbymDUCDuz8d9heRJI0YYmt2EfCsu78d9mOI7e+BV9290d33APcBZ1LE75ySROIZYFSY+e9DUjouzjimXIuBqWF7KslcQNmZmQG3AWvd/QepQ5nHZ2ZDzWxA2D6M5B/KWuBR4JKsYnP3a9y9yt2rSX63HnH3y7KOq5mZHW5mRzZvk4yvryKC19Td3wLWm9kJoek8YE0MsaVcSstQE8QR2xvABDPrH/7NNv/c2v87l+VkT0wPYCLwZ5Ix7G9lHMtdJOOIe0j+JzWNZAz7YWBd+Dooo9j+hqREfR54LjwmxhAfcBKwMsS2CvhOaP8QsByoJxkS6Jvha3s28JuY4gpx/Ck8Vjf//sfwmoY4xgF14XX9NTAwotj6A1uAo1NtscQ2F3gx/Fv4GdC3mN85vS2HiIgUpOEmEREpSElCREQKUpIQEZGClCRERKQgJQkRESlISUJERApSkhARkYL+P8IkRg5nAwBKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "polynomial_regression = Pipeline((\n",
    "    (\"poly_features\", PolynomialFeatures(degree=10, include_bias=False)),\n",
    "    (\"sgd_reg\", LinearRegression()),\n",
    "))\n",
    "\n",
    "plot_learning_curves(polynomial_regression, X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这幅图像和之前的有一点点像，但是其有两个非常重要的不同点：\n",
    "\n",
    "在训练集上，误差要比线性回归模型低的多。\n",
    "图中的两条曲线之间有间隔，这意味模型在训练集上的表现要比验证集上好的多，这也是模型过拟合的显著特点。当然，如果你使用了更大的训练数据，这两条曲线最后会非常的接近。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 线性模型的正则化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 岭（Ridge）回归"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "岭回归（也称为 Tikhonov 正则化）是线性回归的正则化版：在损失函数上直接加上一个正则项 $\\alpha\\sum_{i=1}^m\\theta_i^2$。这使得学习算法不仅能够拟合数据，而且能够使模型的参数权重尽量的小。注意到这个正则项只有在训练过程中才会被加到损失函数。当得到完成训练的模型后，我们应该使用没有正则化的测量方法去评价模型的表现。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "岭回归的封闭方程的解\n",
    "\n",
    "$\\hat{\\theta} = ({\\mathbf{X}}^T\\cdot\\mathbf{X}+\\alpha\\mathbf{A})^{-1}\\cdot{\\mathbf{X}}^T\\cdot\\mathbf{y}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "下面是如何使用 Scikit-Learn 来进行封闭方程的求解（使用 Cholesky 法进行矩阵分解对公式 4-9 进行变形）:\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.1698038]])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "\n",
    "ridge_reg = Ridge(alpha=1, solver=\"cholesky\")\n",
    "ridge_reg.fit(X, y)\n",
    "ridge_reg.predict([[1.5]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "使用随机梯度法进行求解：\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.14465828])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_reg = SGDRegressor(penalty=\"l2\")\n",
    "sgd_reg.fit(X, y.ravel())\n",
    "sgd_reg.predict([[1.5]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "penalty参数指的是正则项的惩罚类型。指定“l2”表明你要在损失函数上添加一项：权重向量 \\ell_2 范数平方的一半，这就是简单的岭回归。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lasso 回归"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lasso 回归（也称 Least Absolute Shrinkage，或者 Selection Operator Regression）是另一种正则化版的线性回归：就像岭回归那样，它也在损失函数上添加了一个正则化项，但是它使用权重向量的 $\\ell_1$ 范数而不是权重向量 $\\ell_2$ 范数平方的一半"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$J(\\theta)=MSE(\\theta)+\\alpha\\sum\\limits_{i=1}^n\\left|\\theta_i \\right|$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lasso 回归的一个重要特征是它倾向于完全消除最不重要的特征的权重（即将它们设置为零）。例如，右图中的虚线所示（$\\alpha=10^{-7}$），曲线看起来像一条二次曲线，而且几乎是线性的，这是因为所有的高阶多项特征都被设置为零。换句话说，Lasso 回归自动的进行特征选择同时输出一个稀疏模型（即，具有很少的非零权重）。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lasso 回归子梯度向量\n",
    "\n",
    "$g(\\theta,J)=\\nabla_{\\theta}MSE(\\theta)+ \\alpha{\\left(\\begin{matrix} sign(\\theta_1)\\\\ sign(\\theta_2)\\\\ \\vdots \\\\ sign(\\theta_n)\\\\ \\end{matrix}\\right)}\\ where\\ sign(\\theta_i)= \\begin{cases} -1, &\\theta_i<0 \\\\ 0, &\\theta_i=0 \\\\ +1,&\\theta_i>0 \\\\ \\end{cases}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "下面是一个使用 Scikit-Learn 的Lasso类的小例子。你也可以使用SGDRegressor(penalty=\"l1\")来代替它。\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.12721696])"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import Lasso\n",
    "lasso_reg = Lasso(alpha=0.1)\n",
    "lasso_reg.fit(X, y)\n",
    "lasso_reg.predict([[1.5]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 弹性网络（ElasticNet）"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "弹性网络介于 Ridge 回归和 Lasso 回归之间。它的正则项是 Ridge 回归和 Lasso 回归正则项的简单混合，同时你可以控制它们的混合率 r，当 r=0 时，弹性网络就是 Ridge 回归，当 r=1 时，其就是 Lasso 回归。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "弹性网络损失函数\n",
    "\n",
    "$J(\\theta)=MSE(\\theta)+r\\alpha\\sum\\limits_{i=1}^n\\left|\\theta_i \\right|+\\frac{1-r}{2}\\alpha\\sum\\limits_{i=1}^n\\theta_i^2$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "一般来说有一点正则项的表现更好，因此通常你应该避免使用简单的线性回归。岭回归是一个很好的首选项，但是如果你的特征仅有少数是真正有用的，你应该选择 Lasso 和弹性网络。就像我们讨论的那样，它两能够将无用特征的权重降为零。一般来说，弹性网络的表现要比 Lasso 好，因为当特征数量比样本的数量大的时候，或者特征之间有很强的相关性时，Lasso 可能会表现的不规律。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "下面是一个使用 Scikit-Learn ElasticNet（l1_ratio指的就是混合率 r）的简单样本："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.12386222])"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import ElasticNet\n",
    "elastic_net = ElasticNet(alpha=0.1, l1_ratio=0.5)\n",
    "elastic_net.fit(X, y)\n",
    "elastic_net.predict([[1.5]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 早期停止法（Early Stopping）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'X_train_poly_scaled' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-76-cbce5668b39d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0mbest_model\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1000\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m     \u001b[0msgd_reg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train_poly_scaled\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      9\u001b[0m     \u001b[0my_val_predict\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msgd_reg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_val_poly_scaled\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m     \u001b[0mval_error\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmean_squared_error\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_val_predict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_val\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'X_train_poly_scaled' is not defined"
     ]
    }
   ],
   "source": [
    "from sklearn.base import clone\n",
    "sgd_reg = SGDRegressor(max_iter=1, warm_start=True, penalty=None,learning_rate=\"constant\", eta0=0.0005)\n",
    "\n",
    "minimum_val_error = float(\"inf\")\n",
    "best_epoch = None\n",
    "best_model = None\n",
    "for epoch in range(1000):\n",
    "    sgd_reg.fit(X_train_poly_scaled, y_train)\n",
    "    y_val_predict = sgd_reg.predict(X_val_poly_scaled)\n",
    "    val_error = mean_squared_error(y_val_predict, y_val)\n",
    "    if val_error < minimum_val_error:\n",
    "        minimum_val_error = val_error\n",
    "        best_epoch = epoch\n",
    "        best_model = clone(sgd_reg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 逻辑回归"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 概率估计"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "逻辑回归模型的概率估计（向量形式）\n",
    "\n",
    "$\\hat{p}=h_\\theta(\\mathbf{x})=\\sigma(\\theta^T  \\cdot \\mathbf{x})$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "逻辑函数\n",
    "\n",
    "$\\sigma(t)=\\frac{1}{1+exp(-t)}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练和损失函数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "单个样本的损失函数\n",
    "\n",
    "$c(\\theta)= \\begin{cases} -log(\\hat{p}), &y=1 \\\\ -log(1-\\hat{p}),&y=0 \\\\ \\end{cases}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "逻辑回归的损失函数（对数损失）\n",
    "\n",
    "$J(\\theta)=-\\frac{1}{m}\\sum\\limits_{i=1}^m\\left[y^{(i)}log\\left(\\hat{p}^{(i)}\\right)+\\left(1-y^{(i)}\\right)log\\left(1-\\hat{p}^{(i)}\\right)\\right]$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先计算每个样本的预测误差，然后误差项乘以第 j 项特征值，最后求出所有训练样本的平均值。 一旦你有了包含所有的偏导数的梯度向量，你便可以在梯度向量上使用批量梯度下降算法。 也就是说：你已经知道如何训练 Logistic 回归模型。 对于随机梯度下降，你当然只需要每一次使用一个实例，对于小批量梯度下降，你将每一次使用一个小型实例集。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 决策边界"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "让我们尝试建立一个分类器，仅仅使用花瓣的宽度特征来识别 Virginica，首先让我们加载数据："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data', 'target', 'target_names', 'DESCR', 'feature_names', 'filename']"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import datasets\n",
    "\n",
    "iris = datasets.load_iris()\n",
    "list(iris.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = iris[\"data\"][:,3:] # petal width\n",
    "y = (iris[\"target\"] == 2).astype(np.int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接下来，我们训练一个逻辑回归模型：\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='auto', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "lr = LogisticRegression()\n",
    "lr.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我们来看看模型估计的花瓣宽度从 0 到 3 厘米的概率估计"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1f410d66e88>"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3gU1dfA8e9NpwRCEpBOooB0CIaiIB0FS0AUaVJEQJGqvCrFQhGRJj8QUGmChSZNUIqgFBEpAQJSFGlCqCGEQALp9/1jQgghCZuwyexuzud55tlyZ2fOsHpy984tSmuNEEII++dkdgBCCCGsQxK6EEI4CEnoQgjhICShCyGEg5CELoQQDsLFrBP7+vpqPz8/s04vhBB2ae/evVe01kXTKzMtofv5+REcHGzW6YUQwi4ppf7LqEyaXIQQwkFIQhdCCAchCV0IIRyEJHQhhHAQktCFEMJB3DehK6XmKaUuK6UOZVCulFLTlFLHlVIHlVK1rR+mEEKI+7Gkhj4faJVJeWugQvLWB/jiwcMSQgiRVffth6613qaU8stklzbAN9qYh3enUspLKVVCa33BSjHe4/vv4fhxcHICZ2djK1oUevY0yletgosX7y1/9lmjfNMmuH79TpmTk1Fep45RvncvJCaCm5uxubtDoULGPgC3boGrq/FZpXLqKoUQjiRJJxGXGEdsQizuLu54uHhY/RzWGFhUCjib6nVo8nv3JHSlVB+MWjxly5bN9gm//x7Wrbv7vUqV7iT0yZNh+/a7ywMD7yT0//s/OHDg7vKmTeG334znHTrAiRN3lwcFwY8/Gs/9/eHSJSOZu7sbSb9TJ/jyS6O8Xj3jsUABYytYEFq1gu7dQWsYN+5OmacnFCkCFSuCn59RnpRk/LEQQuQ+rTWRsZFci7nG9djrmW43Ym9wM+Emt+JvcSvhVqaPsYmxKef48tkveT3wdavHbo2Enl4dNd1VM7TWs4BZAIGBgdleWWPtWiPxJSbe2VKv07F6NcTG3ilLSgKXVFe6bBlER99dXrDgnfL5840afGwsxMUZW8mSd8qHD4fIyDtlsbHw2GN3ysuWNT4fHQ0REcZjxYpGWUwMjBhx7zUNGwaffAJXr4KvLxQubCT6IkXAxwfeeANefBGioozrK17c2EqUAC8v+aUgRGYSkxK5FH2J0OuhnLt+jtDroVyOvsyVm1cIuxnGlZtXUrbwW+EkJCVkejyFopB7IQq6FSSfaz7yueRLefTN73vX69TPPVw8cHdx5/Eyj+fIdVojoYcCZVK9Lg2ct8JxM6WUkaRd0rmCIkUy/2z58pmXN2yYefnAgZmX//BDxmUeHkZSj442ths3jKRfvLhR7uICH31kvBcRYST48HDjMwCnTkGXLncfs0ABmD3b+JVw7hwsWWL8ivD3hwoVjHIhHJnWmotRFzl+9fidLeI4ZyLPEHo9lAs3LpCoE+/6jJNywiefD775ffHN70tFn4o0KNMA3/y++OT3wTufN4XcC6W75XfNj5OyvU6C1kjoq4H+SqnFQD0gMifbz+3d7WYad3fw9r63vHBhGDky488/+igcPWrcI7hwwdjOnr3zCyAkBIYMuft8fn7w3XfwxBPG/ufPQ7VqRgxC2JtLUZc4eOmgsV02Hv8N/5fo+OiUfZyVM/5F/PHz8qO5f3NKFypNKc9SxmOhUpTyLIVvfl+cnRyrbfO+CV0ptQhoAvgqpUKBjwBXAK31l8Ba4BngOHATeDWnghVGe32lSsaWnmeeMWr0p08b9wH+/hsOH77zC2DFCujf37ipW7OmcSO4Th1o3/7uZichbMH12OvsPrebnaE7+TP0T4LPB3M5+nJKeYmCJaj+UHUal2tMee/yVPCuQHnv8pQtXBZXZ1cTIzeHMmuR6MDAQC2zLea+8+dhxw4IDoY9e4zHGzeMewKensb9hQsXjJvEVatK27zIXddjr7P19FY2ndzE5tObOXT5EDr5llyVolWoW6outR6qRfWHqlO9WHWKFkh3FlmHppTaq7UOTK/MtOlzhTlKloSXXjI2MG4Inz5tJHMwavCLFhnPixUzavzt2sHzz5sSrnBwWmsOXT7Eqr9XseHEBnaG7iRRJ5LPJR9PlnuSl6q8RP3S9albqi5eHl5mh2vzpIYu7nH6NGzebPTXX7vW6PK5caNRtmYNNGpktPULkR1aa3ad28WKoytY+fdKjl89jkIRWDKQlg+3pOUjLXm89OO4u8hNnvRkVkOXhC4yFR8PYWFGzf7KFXjoIaP9PSgIunY1+te75r2mSpENZyLP8M2Bb1hwYAHHrx7H1cmVZv7NaFe5HUGPBlG8YHGzQ7QL0uQiss3V9U4ffB8f2LkTvv3WaJb54Qdj9OyiRdC8ublxCtuUmJTImmNrmLFnBr+e/BWNpolfE0Y8OYK2ldpKM4qVSUIXFlPqTq+YyZNhwwaYOxcqVzbKd+0yavT368cvHF9kTCRz989l+u7pnLp2irKFyzKyyUi61uiKfxF/s8NzWJLQRba4usJzzxnbbZ98YoxibdQIPvwQmjWTXjJ5zdVbV/nsz8+YtmsaN+Ju0LBsQya2nEibSm1wcZJ0k9PkX1hYzaJFMGcOjB8PLVoYA5k++QQaNzY7MpHTIm5F8NmfnzF111Si4qJ4qcpLDG04lNolZDbt3GR7Y1eF3cqf35gW4cQJmDHDGMG6f7/ZUYmcFJ8Yz+e7PueRaY/w8e8f06p8Kw72PcjS9kslmZtAaujC6jw84M03jdkvnZKrDN99Z9xQHT06/SkPhP1Zf3w9b294m6NXjtLi4RZMfmoyNR6qYXZYeZrU0EWO8fAwpioAY/76L780bqAuXXr37JjCvlyMukj7H9rT+vvWxCfF82PHH/nllV8kmdsASegiV4wcaUw1UKaMMd98UBCEhpodlcgKrTXfHPiGKjOqsOafNYxtNpZDfQ8R9GgQSu5+2wRJ6CLXBAQYzS6TJhmLiezaZXZEwlJh0WEELQ6i+6ruVC5amZA3Qhj+5HAZzWljJKGLXOXiYkzve/q0sWAHGNMKREWZGpbIxOZTm6n5ZU1+OfELU56ewrYe26jkm8F0n8JUktCFKW6vz3r5MrRpY6z4dPSouTGJuyUmJfLBbx/Q/JvmFHIvxK5euxhcf7DDzSHuSCShC1MVK2ZMABYZaazFenvdVmGuyJhIghYH8fHvH9O9Vnf29tlLreK1zA5L3IckdGG6Jk2MedkffRTatjW6Ngrz/Bv+L/Xn1ueXE7/wxbNf8HWbryngJusY2gPphy5sQunS8PvvxmLY8fFmR5N3bT61mXZL2+GsnNnUdRON/WSYrz2RhC5shocHfP31nT7qf/0FZcvK3Ou5ZfmR5XRe0ZkK3hVY02mNTKJlh6TJRdgUpYzRpbduGXOtN2pkLIknctasvbNo/0N7AksGsu3VbZLM7ZQkdGGT8uUzausnThht7OfOmR2R4xr3+zhe/+l1nqnwDBu7bsQ7n8zNYK8koQub9dRTxpzrFy4YMzaePWt2RI5n3O/jGP7bcLpU78LKDivJ75rf7JDEA5CELmxagwZGUg8Lg48/NjsaxzLxj4kpyXxB2wW4OstagvZObooKm/f44/DHH1C+vNmROI4pf07h3U3v0rFaR+a3nS+DhRyE1NCFXahWzegFEx4OvXvD9etmR2S/vt7/NW//8jYvVXmJb1/4VlYSciCS0IVdOXAA5s83Zmu8dcvsaOzPun/X0XtNb1o+3JLv230vydzBSEIXdqVZM1iwALZuhW7dICnJ7IjsR/D5YNr/0J4aD9Vg+cvLcXN2MzskYWWS0IXd6dwZJk6EZcuMxajF/Z2MOMmzC5/FN78vP3f+GU93T7NDEjlAErqwS0OGQK9esHAh3LhhdjS27UbsDZ5f9DzxifGsf2U9JTxLmB2SyCHSgCbsklIwc6YxS6OnVDYzlKST6LaqG39f+ZsNr2yQecwdnNTQhd1ydQVfX4iNhaFDZYqA9Hy87WNW/b2KSS0n0eLhFmaHI3KYJHRh906dgs8/h5dflpkaU/vx7x/5aMtHdK3RlcH1B5sdjsgFFiV0pVQrpdQ/SqnjSqmh6ZSXVUptVkrtV0odVEo9Y/1QhUhfpUowezZs3w7DhpkdjW04GXGSbqu6EVgykK+e+0oWcc4j7pvQlVLOwAygNVAF6KSUqpJmt/eBpVrrAKAjMNPagQqRmc6doX9/mDzZ6P2Sl8UlxtFhWQeclBPL2i8jn2s+s0MSucSSGnpd4LjW+qTWOg5YDLRJs48GCiU/Lwyct16IQlhm8mSoXx8GDICYGLOjMc/QTUMJPh/M3KC5lPMqZ3Y4IhdZ0sulFJB6nrtQoF6afUYCvyilBgAFgHTvviil+gB9AMqWLZvVWIXIlJsbLFlijCD18DA7GnP8dOwnpuycQv86/WlXuZ3Z4YhcZkkNPb3GN53mdSdgvta6NPAM8K1S6p5ja61naa0DtdaBRW8v+y6EFZUta6xNqrWx4lFecu76Obqv6k6t4rWY+NREs8MRJrAkoYcCZVK9Ls29TSqvAUsBtNZ/Ah6ArzUCFCI7vvgCAgJg506zI8kdWmt6relFTEIMS15agodLHv2JksdZktD3ABWUUv5KKTeMm56r0+xzBmgOoJSqjJHQw6wZqBBZ0aWLsfB0ly55YyTpnH1zWH98PRNaTKCiT0WzwxEmuW9C11onAP2BDcBRjN4sh5VSo5VSQcm7DQF6K6UOAIuAHlrrtM0yQuSawoXhu++MPurvvWd2NDnrVMQp3v7lbZr7N6dvnb5mhyNMpMzKu4GBgTo4ONiUc4u8Y8gQ+Owz2LzZWJvU0STpJJouaMr+C/s59OYhyhaWzgaOTim1V2sdmF6ZzOUiHNqYMUY7+s2bZkeSMz7f9Tnb/tvGvKB5ksyFJHTh2PLnN0aQOuJAyTORZxjx2whal29Nj1o9zA5H2ACZy0U4PKUgIcGYQ33HDrOjsQ6tNf3W9kOjmfnsTBnaLwBJ6CKPuHULpk+HPn0cYwKv5UeX89OxnxjdZDR+Xn5mhyNshCR0kSd4ehozMh4+DFOnmh3Ng4mMiWTguoEEFA9gUP1BZocjbIgkdJFnBAXB88/DyJFw9ux9d7dZw34dxqXoS8x6fpYs8izuIgld5ClTpxoLS7/1ltmRZM/uc7v5MvhLBtYdSGDJdHuuiTxM/ryLPMXf31i6rkIFsyPJuiSdxIB1AyhesDijm442OxxhgyShizynRw+zI8iebw58w+5zu/mm7Td4ustCquJe0uQi8qTEROjXD8aPNzsSy0TGRDJ001AeL/04XWp0MTscYaMkoYs8ydnZWFR6zBj7WFx6zLYxXI6+zLTW03C6d2ZqIQBJ6CIPmzAB4uJgxAizI8nc31f+ZuquqbwW8JrcCBWZkoQu8qzy5WHQIJg/H/btMzuajL214S0KuBZgbPOxZocibJwkdJGnvf8++PjA0KFmR5K+jSc2sv74ej5s/CHFChQzOxxh46SXi8jTCheG77+3zW6MSTqJdze9i5+XH/3q9DM7HGEHJKGLPO+pp+4819p2ZmZc+NdCQi6GsLDdQtxd3M0OR9gBaXIRAoiKgtatjQm8bEFMQgwjfhvBYyUeo0O1DmaHI+yEJHQhgAIFICbG6MZoC2uQTt89nTORZ5jQcoJ0UxQWk/9ShMBoZvn0UwgLM5asM9PVW1cZ+/tYWpdvTTP/ZuYGI+yKJHQhktWrB+3awaRJRmI3y9htY4mMiWR8CzsZxipshiR0IVIZO9ZYf3TSJHPO/9+1/5i+Zzo9avWg+kPVzQlC2C3p5SJEKpUqwfLl0LSpOecfs20MAKOajDInAGHXJKELkUbbtsZjbndhPH71OPND5tOvTj/KFC6TeycWDkOaXIRIx8GDEBAAR4/m3jlHbR2Fm7Mbw54clnsnFQ5FEroQ6ShZEk6cgFG51PJxJOwI3x/8nv51+1O8YPHcOalwOJLQhUiHry8MHAhLl8Jff+X8+UZuGUkBtwK82+DdnD+ZcFiS0IXIwJAhULBgztfSQy6G8MORHxhcbzC++X1z9mTCoUlCFyID3t7GYtLLlxtt6jnlw80f4uXhxZAnhuTcSUSeIL1chMjEW29BuXJQuXLOHH/3ud2sObaGj5t+jJeHV86cROQZktCFyISXF/TsmXPH/2jLR/jk82FgvYE5dxKRZ1jU5KKUaqWU+kcpdVwple5SAEqpl5VSR5RSh5VSC60bphDmmjsXXnvNusfce34v64+vZ8jjQ/B097TuwUWedN+ErpRyBmYArYEqQCelVJU0+1QAhgENtNZVgcE5EKsQprl0CebNg927rXfMT7Z/QmH3wrxZ503rHVTkaZbU0OsCx7XWJ7XWccBioE2afXoDM7TWEQBa68vWDVMIcw0YYNwkHTPGOsc7EnaEFUdXMKDuAAp7FLbOQUWeZ0lCLwWcTfU6NPm91CoCFZVSfyildiqlWqV3IKVUH6VUsFIqOMzM6eyEyCJPT+MG6U8/wf79D368cdvHkd81P4PqD3rwgwmRzJKEnt5sFjrNaxegAtAE6ATMUUrdc8teaz1Lax2otQ4sWrRoVmMVwlT9+0OhQvDJJw92nJMRJ1n01yLeeOwN6XcurMqSXi6hQOqZgkoD59PZZ6fWOh44pZT6ByPB77FKlELYAC8v+N//oFTa36dZNH77eJydnKXfubA6S2roe4AKSil/pZQb0BFYnWafVUBTAKWUL0YTzElrBiqELXj11bsXlc6qc9fPMf/AfHrW6klJz5LWC0wILKiha60TlFL9gQ2AMzBPa31YKTUaCNZar04ue0opdQRIBN7RWofnZOBCmOXKFWO5ujfegPLls/bZSTsmkZiUaNU5W+Lj4wkNDSUmJsZqxxTm8/DwoHTp0ri6ulr8GaV12ubw3BEYGKiDg4NNObcQD+LCBfD3h1degTlzLP9cWHQY5f5XjvZV27Og7QKrxXPq1Ck8PT3x8fFB5eYE7iLHaK0JDw/nxo0b+Pv731WmlNqrtQ5M73Myl4sQWVSiBPTqBQsWwJkzln/ufzv/R0xCDEMbpDs2L9tiYmIkmTsYpRQ+Pj5Z/tUlCV2IbHj3XWM1owkTLNs/MiaS6Xum065yOyoXtf7EMJLMHU92vlNJ6EJkQ9my0L270eRy4cL995+xZwbXY68z4skROR+cCQoWLJhh2RNPPGHRMaKjo/Hx8SEyMvKu99u2bcvSpUtZvXo1n376aZZjs+T8vXr14siRI1k+tq2RybmEyKahQyEqCuLjM98vOi6aKTun0Lp8awJKBOROcDYgMTERZ2dnduzYYdH+BQoU4KmnnmLVqlV0794dgMjISLZv387ChQvJnz8/QUFB93wuISEBF5eMU5kl55+TlZshNkxq6EJk0yOPwKJFRm09M7P3zebKzSsOWztPbcuWLTRt2pTOnTtTvXp14E7t/cKFCzRq1IhatWpRrVo1fv/993s+36lTJxYvXpzyeuXKlbRq1Yr8+fMzf/58+vfvD0CPHj14++23adq0Ke+99x5hYWG0bNmS2rVr8/rrr1OuXDmuXLly1/m3bNlCkyZNeOmll6hUqRJdunThdqeQJk2acLuTxvr166lduzY1a9akefPmAOzevZsnnniCgIAAnnjiCf7555+c+Od7YFJDF+IBHToEf/8NL710b1lsQiwTd0ykcbnGNCjbIMdjGbx+MCEXQ6x6zFrFa/G/Vv+zeP/du3dz6NChe3pnLFy4kKeffpoRI0aQmJjIzZs37/lsq1at6NWrF+Hh4fj4+LB48WIGDBiQ7nmOHTvGpk2bcHZ2pn///jRr1oxhw4axfv16Zs2ale5n9u/fz+HDhylZsiQNGjTgjz/+oGHDhinlYWFh9O7dm23btuHv78/Vq1cBqFSpEtu2bcPFxYVNmzYxfPhwli9fbvG/SW6RhC7EAxo5EjZtgpYtoXCaebYWHFjA+Rvnmd9mvhmhmaJu3br3JHOAOnXq0LNnT+Lj42nbti21atW6Zx83NzeCgoJYtmwZL774IiEhITyVwUiu9u3b4+zsDMD27dtZuXIlYPxRKFKkSIaxlS5dGoBatWpx+vTpuxL6zp07adSoUUr83t7egNH00717d/7991+UUsTfr53NJJLQhXhAw4YZy9TNnGk8vy0hKYHxf4ynTsk6tHi4Ra7EkpWadE4pUKBAuu83atSIbdu28fPPP9O1a1feeecdPD09GZW8aOucOXMIDAykU6dOfPzxx2itadOmTYYDa1Kfx9LxNO7u7inPnZ2dSUhIuKtca51u75IPPviApk2bsnLlSk6fPk2TJk0sOl9ukzZ0IR7QY49Bq1YwZQqkbkVYfGgxJyNOMuLJEdKtEPjvv/8oVqwYvXv35rXXXmPfvn288MILhISEEBISQmCgMVamadOm/Pvvv8yYMYNOnTpZdOyGDRuydOlSAH755RciIiKyFePjjz/O1q1bOXXqFEBKk0tkZCSlkifxmT9/fraOnRskoQthBSNGQFjYnZGjSTqJcdvHUa1YNZ5/9Hlzg7MRW7ZsoVatWgQEBLB8+XIGDUp/6mAnJydefPFFwsPDadSokUXH/uijj/jll1+oXbs269ato0SJEnh6Zn0VqKJFizJr1izatWtHzZo16dChAwDvvvsuw4YNo0GDBiQmJmb5uLlFhv4LYSXPPgtPPml0Z1xxdAUvLn2Rhe0W0qm6ZbXM7Dp69CiVc2oVazsRGxuLs7MzLi4u/Pnnn/Tt25eQEOveHDZDet9tZkP/pQ1dCCv56Sdj9KjWmk9+/4RHijxC+6rtzQ4rTzhz5gwvv/wySUlJuLm5MXv2bLNDMoUkdCGsxEjmMPH7YPaGhjC77Ze4OMn/YrmhQoUK7LfGUlJ2TtrQhbCiX3+F97rWwftkX7rV7GZ2OCKPkeqDEFbk+sjvULQIbn+OxEW5mR2OyGOkhi6EFY37YyyezWdw8aQPq9Ou6yVEDpOELoSVBJ8PZsOJDbz3uh8PPwxjxxpt6kLkFknoQljJJ79/gpeHFwMe78vQoRAaamx5gVKKIUPuLHo9adIkRo4cmelnVq1aleGUtSNHjmTSpEnpln355Zd88803FsU1cuRIhqUevguEhISkdAV85plnuHbtmkXHysr5g4ODGThwYJaOaw2S0IWwgsOXD7Py75UMqDuAQu6F6N4dTp2CMmXMjix3uLu7s2LFipQZDi2RWULPSEJCAm+88Qbdull2w7lTp04sWbLkrvcWL15M586dAVi7di1eXl53lWutSUpKyvCYlpw/MDCQadOmWRSjNUlCF8IKxm0fRwHXAgyqZ4x+dHMDDw9jrvSLF00OLhe4uLjQp08fpkyZck/Zf//9R/PmzalRowbNmzfnzJkz7Nixg9WrV/POO+9Qq1YtTpw4keGxmzRpwvDhw2ncuDFTp069q/Y+bdo0qlSpQo0aNejYseM9n3300Ufx8vJi165dKe8tXbo0ZV8/Pz+uXLnC6dOnqVy5Mm+++Sa1a9fm7NmzzJ07l4oVK9KkSRN69+6dMnVv6vM3adKE9957j7p161KxYsWUKYG3bNnCc889B0BUVBSvvvoq1atXp0aNGimzNPbt25fAwECqVq3KRx99lOV/8/RILxchHtCJqydYdGgRb9V/C5/8Pinvaw0NG4K3N6xbl3vxpDdv1Msvw5tvGnPNPPPMveU9ehjblSv3TgO8ZYtl5+3Xrx81atTg3Xffvev9/v37061bN7p37868efMYOHAgq1atIigoiOeee46X0pt3OI1r166xdetWgLuacj799FNOnTqFu7t7hk0nt+dYr1evHjt37sTHx4cKFSrcs98///zD119/zcyZMzl//jxjxoxh3759eHp60qxZM2rWrJnu8RMSEti9ezdr165l1KhRbNq06a7yMWPGULhwYf766y+AlHlmxo4di7e3N4mJiTRv3pyDBw9So0aN+/5bZEZq6EI8oPF/jMfFyYW3H3/7rveVgrZtYf162LvXpOByUaFChejWrds9TQ1//vlnShNH165d2b59e5aPfXtOlbRq1KhBly5d+O677zJctahjx44sW7aMpKQkFi9enOGEX+XKlaN+/fqAMad748aN8fb2xtXVlfbtMx7x265dOwAee+wxTp8+fU/5pk2b6NevX8rr21P7Ll26lNq1axMQEMDhw4etsgSe1NCFeADnrp9jfsh8Xgt4jZKeJe8pf/NNGD8ePvnEmGI3N2RWo86fP/NyX1/La+TpGTx4MLVr1+bVV1/NcJ/szDyZ0ZS8P//8M9u2bWP16tWMGTOGw4cP8+yzz3Lp0iUCAwOZM2cOZcqUwc/Pj61bt7J8+XL+/PPP+54jK3Nc3Z6SN73peG8fK+01nzp1ikmTJrFnzx6KFClCjx49iImJsficGZEauhAPYOKOiSTpJN5t8G665YULw4ABsGIFOMAaxPfl7e3Nyy+/zNy5c1Pee+KJJ1KWlfv+++9TFpTw9PTkxo0b2T5XUlISZ8+epWnTpkyYMIFr164RFRXFhg0bCAkJuWud0E6dOvHWW2/xyCOPpCxwkZm6deuydetWIiIiSEhIeKDViZ566immT5+e8joiIoLr169ToEABChcuzKVLl1hnpTY5SehCZNOlqEvM2juLrjW74l/k3hV6bhs0yKgZW9jTzu4NGTLkrt4u06ZN4+uvv6ZGjRp8++23TJ06FTCaQiZOnEhAQECmN0UzkpiYyCuvvEL16tUJCAjgrbfeuqfHym3t27fn8OHD6d44TU+pUqUYPnw49erVo0WLFlSpUoXCaZejstD7779PREQE1apVo2bNmmzevJmaNWsSEBBA1apV6dmzJw0aWGd5Qpk+V4hsGrppKBP+mMDf/f+mok/FTPc9cgQqVQKnHKhCyfS5OSMqKoqCBQuSkJDACy+8QM+ePXnhhRdyNYasTp8rNXQhsuHqravM2DODDtU63DeZA1SpYiRzG12KUqRj5MiR1KpVi2rVquHv70/btm3NDum+5KaoENkwbdc0ouKiGN5wuMWfWbMGevc2erwkr2YmbFhGI1VtmdTQhcii67HXmbprKm0rtaX6Q9Ut/ly1akY/78mTczA4kadJQhcii2bumcm1mGuMeHJElj7n7w+dO8NXXxmJ3ZrMumFCI2sAABa+SURBVBcmck52vlNJ6EJkQXRcNJP/nEyr8q0ILJnufalMDRsGt27B//5nvZg8PDwIDw+XpO5AtNaEh4fj4eGRpc9Z1IaulGoFTAWcgTla608z2O8l4AegjtZaurAIhzN732yu3LzC+0++n63PV64M7drB9OlGcs9gvEyWlC5dmtDQUMLCwh78YMJmeHh4WNRnPrX7dltUSjkDx4CWQCiwB+iktT6SZj9P4GfADeh/v4Qu3RaFvYlJiOGRaY9Q0acim7tvzvZx/v4brl+HunWtGJzIMzLrtmhJDb0ucFxrfTL5YIuBNkDacW9jgAnA/z1ArELYrPkh8zl/4zzftH2wEUKVKlkpICHSsKQNvRRwNtXr0OT3UiilAoAyWuufMjuQUqqPUipYKRUsPw+FPYlLjOPT7Z9Sv3R9mvk3e+DjxcbCa6/BzJlWCE6IZJYk9PRm0klpp1FKOQFTgCHp7Hf3h7SepbUO1FoHFi1a1PIohTDZ/JD5/Bf5Hx80+iBbk0ul5e4Ox48bk3bFxlohQCGwLKGHAqnXXSkNnE/12hOoBmxRSp0G6gOrlVJZ7wIghA2KTYhl7O9jqVeqHq3Lt7bacUeMgHPn8s4cLyLnWZLQ9wAVlFL+Sik3oCOQsp651jpSa+2rtfbTWvsBO4Eg6eUiHMW8/fM4E3mG0U1HW6V2flvLllCnDowbJ1MCCOu4b0LXWicA/YENwFFgqdb6sFJqtFIqKKcDFMJMMQkxjP19LA3KNKDlwy2temylYORIY+3Rr7+26qFFHmVRP3St9VpgbZr3Psxg3yYPHpYQtmH23tmcu3GOBW0XWLV2flvr1kY7eqtWVj+0yINkci4hMnAr/hafbP+ERuUaWaVnS3qUMgYYCWENMvRfiAx8tfcrLkZdZHQT67adp+fgQejY0VjEWYjskoQuRDqi46IZt30czfyb0divcY6f78YNWLIEZszI8VMJByYJXYh0fBH8BZejLzOqyahcOV+DBkY7+vjxRnIXIjskoQuRRmRMJOO2j+OpR56iYdmGuXbe0aMhPBySl9wUIsskoQuRxoQ/JnD11lU+bZ7upKI5pk4dCAqCSZMgIiJXTy0chPRyESKVCzcuMGXnFDpW60hAiYBcP//o0bB6tTE1gBBZJQldiFRGbx1NfFI8Y5qOMeX8NWsamxDZIU0uQiT7N/xfZu+bTZ/afSjvXd7UWFatgg/THbonRMYkoQuR7IPNH+Du4s4HjT8wOxR27ICPPzb6pwthKUnoQgB7z+9lyeElvF3/bYoXLG52OAwbBl5e8N57Zkci7IkkdJHnaa15Z+M7+OTz4f+esI0Ft4oUMabXXb8eNm0yOxphLyShizzvx39+ZPPpzYxuOprCHoXNDidFv35Qrhy8+y4kJZkdjbAH0stF5GmxCbH83y//R5WiVejzWB+zw7mLh4cxyOj6dbMjEfZCErrI06bvns6JiBOs77IeFyfb+9+hTRuzIxD2RJpcRJ4VFh3G6G2jeabCMzxd/mmzw8mQ1vDZZ/Bp7g5cFXZIErrIsz7c/CHRcdFMajnJ7FAypRTs22esbnTypNnRCFsmCV3kSQcvHWTWvlm8WedNKhetbHY49zV+PLi4wJAhZkcibJkkdJHnJOkk+v7cF+983oxsMtLscCxSqhS8/74xgnTjRrOjEbZKErrIcxaELGDH2R1MaDEB73zeZodjsbfegkcegcGDpRujSJ/t3dYXIgeF3wznnY3v0KBMA7rX6m52OFni7g5z5hhNL05SFRPpkIQu8pThvw7nWsw1Zj47Eydlf1mxSZM7zxMTwdnZtFCEDbK//6KFyKadoTuZvW82g+oNosZDNcwO54GMGAHPP290aRTiNknoIk+IT4yn7899KeFZwm5uhGbmoYdg3TpYtMjsSIQtkYQu8oSJOyYScjGEz1t/jqe7p9nhPLB+/Ywl6wYPhqtXzY5G2ApJ6MLhHQk7wqito2hfpT3tKrczOxyrcHaGWbOMZP7OO2ZHI2yFJHTh0BKTEun5Y0883TyZ/sx0s8Oxqlq1jIFGixbBuXNmRyNsgSR04dCm7prKrnO7+Lz15xQrUMzscKxu1CgICTEGHgkhCV04rH+u/MOI30YQ9GgQHat1NDucHOHhARUrGr1dduwwOxphNknowiHFJcbRZUUX8rvm54tnv0ApZXZIOWrJEmjQAFauNDsSYSaLErpSqpVS6h+l1HGl1NB0yt9WSh1RSh1USv2qlCpn/VCFsNzILSPZe2Evc56fQ0nPkmaHk+PatYOAAHj9dbh82exohFnum9CVUs7ADKA1UAXopJSqkma3/UCg1roGsAyYYO1AhbDU1tNb+XT7p/QK6MULlV8wO5xc4eYG33xjrG7Uo4fM9ZJXWVJDrwsc11qf1FrHAYuBu9ZR0Vpv1lrfTH65Eyht3TCFsEzErQi6ruxKee/yTGk1xexwclW1ajBlijHgaPJks6MRZrBkLpdSwNlUr0OBepns/xqwLr0CpVQfoA9A2bJlLQxRCMtorem1phcXoi6wo+cOCroVNDukXPfGG7B/v3GjVOQ9liT09O4mpTuDhFLqFSAQaJxeudZ6FjALIDAwUGahEFY1+c/JrDi6gkktJ1GnVB2zwzGFUsaAo9u0Nt4TeYMlTS6hQJlUr0sD59PupJRqAYwAgrTWsdYJTwjLbD29laGbhvJi5Rd5+/G3zQ7HJkydCi+/LO3peYklCX0PUEEp5a+UcgM6AqtT76CUCgC+wkjmco9d5KoLNy7QYVkHynuXZ16beQ7fRdFSTk6wbJkx+EjkDfdtctFaJyil+gMbAGdgntb6sFJqNBCstV4NTAQKAj8k/890RmsdlINxCwFAbEIsLy97mRtxN/i1268Uci9kdkg2o39/Y3Hp0aOhZk2ja6NwbBYtcKG1XgusTfPeh6met7ByXELcl9aa1396ne1ntrP4xcVULVbV7JBsilLwxRdw5Ah062bcKK1WzeyoRE6SkaLCbo3/YzwLDixgZOORdKjWwexwbJKHhzF61MtLpgbIC2QJOmGXlh9ZzrBfh9GpWic+bPzh/T+Qh5UsCUePgqf9TwMv7kNq6MLubD+znVdWvkL90vXlJqiFbifzX3+Fzp0hIcHceETOkIQu7MqBiwd4buFzlC1cltUdV+Ph4mF2SHbl+HFj/vS+fWU9UkckTS7Cbhy/epynv3saT3dPNnbdSNECRc0Oye68/jqcPQtjx4KPD4wbJwOPHIkkdGEX/rv2Hy2/bUlCUgKbu2+mbGGZOiK7xoyB8HAYPx5cXIzXktQdgyR0YfNORZyi6YKmRMZGsrHrRioXrWx2SHZNKZgxw2hHDw2V6QEciSR0YdNORpyk6YKm3Ii9waaum3is5GNmh+QQnJzgq6/uPL96FYoUkcRu7+SmqLBZR8KO0Hh+Y6Liovi126+SzK3MyelOMq9TBwYNknlf7J0kdGGTtp/ZTsN5DVPazANKBJgdksPy8oKgIPj8c+jaFeLizI5IZJc0uQibs/LoSjqv6Ey5wuVY/8p6/Lz8zA7JoTk5wWefwUMPwbBhxg3TpUuhkEyLY3ekhi5shtaaT7d/yotLX6RW8Vps77ldknkuUQqGDoU5c2DTJnhbZiC2S1JDFzYhOi6anqt7svTwUjpW68jcoLnkd81vdlh5zmuvQfnyUCV51eCkJKMGL+yDfFXCdCeunuCJeU+w7MgyJrSYwMJ2CyWZm6hxYyhaFOLj4emnYcIEuVlqLyShC9Norfn2wLfU+qoWZyLPsLbzWt5p8I7MzWIj4uKMG6bvvQetW8OlS2ZHJO5HErowRWRMJF1WdKHbqm4EFA/gwBsHeLr802aHJVIpUMC4Ofrll7BtG9SoAevXmx2VyIwkdJHrfj72M9W/qM7Sw0sZ03SMDOW3YUoZ87/s2WM0wwwYYDTFCNskN0VFrrkUdYnBGwaz+NBiqhStwvb226lfur7ZYQkLVKsGwcHGxF6urnDzJvz2Gzz7rIwutSVSQxc5Lj4xnmm7plF5RmVWHF3BqCaj2P/6fknmdsbDAypUMJ7PnAnPPw+tWhlL3AnbIDV0kWO01qw5toZ3Nr7DsfBjNPNvxvTW02VyLQcwaJAxU+OoUUbbet++MHKkMSWvMI/U0IXVaa3ZfGozTRc0pc3iNigUP3X6iU1dN0kydxCurjB4MPz7r9HGPnMm9OljdlRCaujCarTWbDy5kdFbR/PH2T8oUbAE01tPp89jfXB1djU7PJEDfH2NqXjffPNOW/qpUzB/PgwcKDX23CY1dPHAbsXfYu6+uQR8FcDT3z3Nf5H/Mb31dE4OOkm/uv0kmecBVaveGV26fj2MHg2lS0Pv3nDokLmx5SWS0EW2/Rv+L+9tfI/SU0rTa00vEnUis56bxfEBx+lXt5+s95lH9e1rJPGuXeG776B6dWjTRtYwzQ3S5CKyJPxmOEsOL+Hbg9+yM3QnzsqZtpXaMqDuABqVaySjPAVg1NhnzTLWLJ09GyIi7jTJfPwxtGgB9epJl0drU9qkP5uBgYE6ODjYlHOLrDl/4zxr/lnD6mOr2XhiI/FJ8VQvVp2uNbrSuXpnShUqZXaIwk6EhkLFinDrFpQtCy+8YGwNG4Kzs9nR2Qel1F6tdWC6ZZLQRVrxifEEnw9m08lNrD62muDzxvf0cJGHaVepHa/UeIWaxWuaHKWwV9evw4oVxvbLLxAbC4sWQceOxlzscXFQooTZUdouSegiU7EJsRy4dIAtp7ew+fRmtp/ZTlRcFApFvdL1CKoYRNCjQVQpWkWaVIRVRUXBunXGrI6FCsHkyfB//2c02TRvDg0aQP36UKaMNM/cJgldpIhPjOdY+DH2nN/DnnN72HN+DwcuHSAu0Vh3rLJvZZr6NaWpf1Mal2tM0QJFTY5Y5CXHjsGPPxqLbPz+u9E04+Ji1Orz5TOmG4iPNwYzFS+eN5O8JPQ8KDoumlPXTnE07ChHwo5wOOwwR8KOcCz8GPFJxuxKnm6eBJYMpE7JOtQpVYeGZRtSvGBxkyMXwhAfDwcPGoOXOnY03mvWDDZvNp77+ho9aJo0gQ8/NN4LCwNvb8duj88soUsvFzuUmJTI5ejLXIy6yMWoi5yJPMOpa6c4de0Up6+d5lTEKcJuhqXsr1A8XORhqharyvMVn6dqsaoElgykok9FnJT0XBW2ydUVHnvM2G5btgz++stI9Le3kJA75Q0bwunT8PDDxspLfn7Gex06GOWnT0OxYpDfQddPsaiGrpRqBUwFnIE5WutP05S7A98AjwHhQAet9enMjik1dENiUiLXYq5x9dZVImIiiLgVkfJ4+73wW+EpyfvCjQuE3QwjSd+9hIyrkyvlvMrh5+WHv5c//l7++Hn5Ucm3EpV8K5HPNZ9JVyhE7pk/H44ehePHjZr92bNGH/j5841+8PnzQ0wMFCliLIpdtCh06mT0nU9KgqlTjZq/r6+xuEehQlCypLG/rXigGrpSyhmYAbQEQoE9SqnVWuvUc6y9BkRorcsrpToC44EODx669WmtSdSJJOkkEpMSSdSJdz0m6STik+KJS4xL2WITYu88T4zN9P3YhFhuxt8kOj6aqLgoouOjiY6LvusxKi4q5fnN+JuZxpvPJR/e+bwp4VmC0oVKE1gikBKeJShesDglChqPpQuVpqRnSZydHPh3phAW6NHj3vcSEozHpCT46is4d87oPnn5stFEc3t+92vX0l8ce9Qoo0nn/HmoVMlI8re3ggWhf39o2xYuXDBGyObLZ8xMmS+fsbVubdzkvXoVtm41umum/tVhTZY0udQFjmutTwIopRYDbYDUCb0NMDL5+TJgulJK6RxooJ+3fx4Td0zMNCFn9p4m5+8ZOCtnCrgVoIBrAQq6FUx5XtijMCU9SxrvuRaggFsBPN08KZKvCEU8ilAkXxG883mnPC/iUQR3F/ccj1cIR+aSnOWcnaFbt4z3K1LEGAB15YqR6CMjjZuxt6c0cHMzFtG+fv3OduPGnT8IV68aXTFv3TK2239IHnrISOiHD0O7dkbzz+LFOXStFuxTCjib6nUoUC+jfbTWCUqpSMAHuJJ6J6VUH6APQNmy2Vuhxje/L9WLVcfZyRkn5YSzcsbZydl4VMnv3X6d6vGefTN5z8XJBXdnd9xd3HFzdsPN2Q135zvP3ZzdMixzd3HH1clVuvcJYWeUMppZvLyM9ve0fH1hypSMP1+16t3rriYkGIndzc14HRAA+/YZNfucYklCTy8zpa3mWrIPWutZwCww2tAtOPc9gh41+kQLIYQtc3EBT887rwsWNJJ6TrKki0MoUCbV69LA+Yz2UUq5AIWBq9YIUAghhGUsSeh7gApKKX+llBvQEVidZp/VQPfk5y8Bv+VE+7kQQoiM3bfJJblNvD+wAaPb4jyt9WGl1GggWGu9GpgLfKuUOo5RM++Yk0ELIYS4l0UDi7TWa4G1ad77MNXzGKC9dUMTQgiRFTJMUAghHIQkdCGEcBCS0IUQwkFIQhdCCAdh2vS5Sqkw4L9sftyXNKNQ7Zhci+1xlOsAuRZb9SDXUk5rne5CBaYl9AehlArOaLYxeyPXYnsc5TpArsVW5dS1SJOLEEI4CEnoQgjhIOw1oc8yOwArkmuxPY5yHSDXYqty5Frssg1dCCHEvey1hi6EECINSehCCOEgbDqhK6VaKaX+UUodV0oNTafcXSm1JLl8l1LKL/ejtIwF19JDKRWmlApJ3nqZEef9KKXmKaUuK6UOZVCulFLTkq/zoFKqdm7HaCkLrqWJUioy1XfyYXr7mU0pVUYptVkpdVQpdVgpNSidfezie7HwWuzle/FQSu1WSh1IvpZR6exj3RymtbbJDWOq3hPAw4AbcACokmafN4Evk593BJaYHfcDXEsPYLrZsVpwLY2A2sChDMqfAdZhrGJVH9hldswPcC1NgJ/MjtOC6ygB1E5+7gkcS+e/L7v4Xiy8Fnv5XhRQMPm5K7ALqJ9mH6vmMFuuoacsTq21jgNuL06dWhtgQfLzZUBzZZuLeVpyLXZBa72NzFejagN8ow07AS+lVInciS5rLLgWu6C1vqC13pf8/AZwFGOd39Ts4nux8FrsQvK/dVTyS9fkLW0vFKvmMFtO6OktTp32i71rcWrg9uLUtsaSawF4Mfnn8DKlVJl0yu2BpddqLx5P/sm8TilV1exg7if5J3sARm0wNbv7XjK5FrCT70Up5ayUCgEuAxu11hl+L9bIYbac0K22OLUNsCTONYCf1roGsIk7f7Xtjb18J5bYhzFvRk3gc2CVyfFkSilVEFgODNZaX09bnM5HbPZ7uc+12M33orVO1FrXwliLua5SqlqaXaz6vdhyQnekxanvey1a63CtdWzyy9nAY7kUm7VZ8r3ZBa319ds/mbWxaperUsrX5LDSpZRyxUiA32utV6Szi918L/e7Fnv6Xm7TWl8DtgCt0hRZNYfZckJ3pMWp73stadozgzDaDu3RaqBbcq+K+kCk1vqC2UFlh1Kq+O32TKVUXYz/X8LNjepeyTHOBY5qrT/LYDe7+F4suRY7+l6KKqW8kp/nA1oAf6fZzao5zKI1Rc2gHWhxaguvZaBSKghIwLiWHqYFnAml1CKMXga+SqlQ4COMmz1orb/EWHv2GeA4cBN41ZxI78+Ca3kJ6KuUSgBuAR1ttMLQAOgK/JXcXgswHCgLdve9WHIt9vK9lAAWKKWcMf7oLNVa/5STOUyG/gshhIOw5SYXIYQQWSAJXQghHIQkdCGEcBCS0IUQwkFIQhdCCAchCV0IIRyEJHQhhHAQ/w9u/bEjIx1EVgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_new = np.linspace(0, 3, 1000).reshape(-1, 1)\n",
    "y_proba = lr.predict_proba(X_new)\n",
    "plt.plot(X_new, y_proba[:, 1], \"g-\", label=\"Iris-Virginica\")\n",
    "plt.plot(X_new, y_proba[:, 0], \"b--\", label=\"Not Iris-Virginica\")\n",
    "plt.legend(loc = 'best')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Virginica 花的花瓣宽度（用三角形表示）在 1.4 厘米到 2.5 厘米之间，而其他种类的花（由正方形表示）通常具有较小的花瓣宽度，范围从 0.1 厘米到 1.8 厘米。注意，它们之间会有一些重叠。在大约 2 厘米以上时，分类器非常肯定这朵花是 Virginica 花（分类器此时输出一个非常高的概率值），而在 1 厘米以下时，它非常肯定这朵花不是 Virginica 花（不是 Virginica 花有非常高的概率）。在这两个极端之间，分类器是不确定的。但是，如果你使用它进行预测（使用predict()方法而不是predict_proba()方法），它将返回一个最可能的结果。因此，在 1.6 厘米左右存在一个决策边界，这时两类情况出现的概率都等于 50%：如果花瓣宽度大于 1.6 厘米，则分类器将预测该花是 Virginica，否则预测它不是（即使它有可能错了）："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 0])"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.predict([[1.7], [1.5]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Softmax 回归"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Logistic 回归模型可以直接推广到支持多类别分类，不必组合和训练多个二分类器（如第 3 章所述）， 其称为 Softmax 回归或多类别 Logistic 回归。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "这个想法很简单：当给定一个实例 $\\mathbf{x}$ 时，Softmax 回归模型首先计算 k 类的分数 $s_k(\\mathbf{x})$，然后将分数应用在Softmax 函数（也称为归一化指数）上，估计出每类的概率。 计算 $s_k(\\mathbf{x})$ 的公式看起来很熟悉，因为它就像线性回归预测的公式一样（见公式 4-19）。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "k类的 Softmax 得分\n",
    "\n",
    "$s_k(\\mathbf{x})= \\theta^T  \\cdot \\mathbf{x}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "注意，每个类都有自己独一无二的参数向量 $\\theta_k$。 所有这些向量通常作为行放在参数矩阵$ \\Theta $中。\n",
    "\n",
    "一旦你计算了样本 $\\mathbf{x} $的每一类的得分，你便可以通过Softmax函数估计出样本属于第 $k$ 类的概率 $\\hat{p}_k$：通过计算 $e $的 $s_k(\\mathbf{x})$ 次方，然后对它们进行归一化（除以所有分子的总和）。\n",
    "\n",
    "Softmax 函数\n",
    "\n",
    "$\\hat{p_k}=\\sigma{(\\mathbf{s}(\\mathbf{x}))}k= \\frac{exp\\left(s_k(\\mathbf{x})\\right)} {\\sum_{j=1}^{K}exp\\left(s_j(\\mathbf{x})\\right)}$\n",
    "\n",
    "K 表示有多少类\n",
    "$\\mathbf{s}(\\mathbf{x})$ 表示包含样本 $\\mathbf{x}$ 每一类得分的向量\n",
    "$\\sigma{(\\mathbf{s}(\\mathbf{x}))_k} $表示给定每一类分数之后，实例 $\\mathbf{x}$ 属于第$ k $类的概率\n",
    "和 Logistic 回归分类器一样，Softmax 回归分类器将估计概率最高（它只是得分最高的类）的那类作为预测结果\n",
    "\n",
    "Softmax 回归模型分类器预测结果\n",
    "\n",
    "$\\hat{y}=argmax\\ \\sigma{(\\mathbf{s}(\\mathbf{x}))_k}=argmax \\ s_k(\\mathbf{x})=argmax \\ \\left( \\theta_k^T  \\cdot \\mathbf{x}\\right)$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "交叉熵\n",
    "\n",
    "$J(\\Theta)=-\\frac{1}{m}\\sum\\limits_{i=1}^m\\sum\\limits_{k=1}^Ky_k^{(i)}log\\left(\\hat{p}_k^{(i)}\\right)$\n",
    "\n",
    "如果对于第$ i $个实例的目标类是 $k$，那么 $y_k^{(i)}=1$，反之 $y_k^{(i)}=0$。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "k类交叉熵的梯度向量\n",
    "\n",
    "$\\nabla_{\\theta_k}J(\\Theta)=\\frac{1}{m}\\sum\\limits_{i=1}^m\\left(\\hat{p}_k^{(i)}-y_k^{(i)}\\right)\\mathbf{x}^{(i)}$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "让我们使用 Softmax 回归对三种鸢尾花进行分类。当你使用LogisticRregression对模型进行训练时，Scikit Learn 默认使用的是一对多模型，但是你可以设置multi_class参数为“multinomial”来把它改变为 Softmax 回归。你还必须指定一个支持 Softmax 回归的求解器，例如“lbfgs”求解器（有关更多详细信息，请参阅 Scikit-Learn 的文档）。其默认使用 \\ell_12 正则化，你可以使用超参数 C 控制它。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = iris[\"data\"][:, (2, 3)] # petal length, petal width\n",
    "y = iris[\"target\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=10, class_weight=None, dual=False, fit_intercept=True,\n",
       "                   intercept_scaling=1, l1_ratio=None, max_iter=100,\n",
       "                   multi_class='multinomial', n_jobs=None, penalty='l2',\n",
       "                   random_state=None, solver='lbfgs', tol=0.0001, verbose=0,\n",
       "                   warm_start=False)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax_reg = LogisticRegression(multi_class=\"multinomial\",solver=\"lbfgs\", C=10)\n",
    "softmax_reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "预测分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2])"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "softmax_reg.predict([[5, 2]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对应分类的对应概率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[6.38014896e-07, 5.74929995e-02, 9.42506362e-01]])"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " softmax_reg.predict_proba([[5, 2]])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
